{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use DDPG to Play Pendulum-v0\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import losses\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import models\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:22:27 [INFO] env: <PendulumEnv<Pendulum-v0>>\n",
      "22:22:27 [INFO] action_space: Box(-2.0, 2.0, (1,), float32)\n",
      "22:22:27 [INFO] observation_space: Box(-8.0, 8.0, (3,), float32)\n",
      "22:22:27 [INFO] reward_range: (-inf, inf)\n",
      "22:22:27 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 30}\n",
      "22:22:27 [INFO] _max_episode_steps: 200\n",
      "22:22:27 [INFO] _elapsed_steps: None\n",
      "22:22:27 [INFO] id: Pendulum-v0\n",
      "22:22:27 [INFO] entry_point: gym.envs.classic_control:PendulumEnv\n",
      "22:22:27 [INFO] reward_threshold: None\n",
      "22:22:27 [INFO] nondeterministic: False\n",
      "22:22:27 [INFO] max_episode_steps: 200\n",
      "22:22:27 [INFO] _kwargs: {}\n",
      "22:22:27 [INFO] _env_name: Pendulum\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Pendulum-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['state', 'action', 'reward', 'next_state', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "\n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OrnsteinUhlenbeckProcess:\n",
    "    def __init__(self, x0):\n",
    "        self.x = x0\n",
    "\n",
    "    def __call__(self, mu=0., sigma=1., theta=.15, dt=.01):\n",
    "        n = np.random.normal(size=self.x.shape)\n",
    "        self.x += (theta * (mu - self.x) * dt + sigma * np.sqrt(dt) * n)\n",
    "        return self.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DDPGAgent:\n",
    "    def __init__(self, env):\n",
    "        state_dim = env.observation_space.shape[0]\n",
    "        self.action_dim = env.action_space.shape[0]\n",
    "        self.action_low = env.action_space.low\n",
    "        self.action_high = env.action_space.high\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = DQNReplayer(20000)\n",
    "\n",
    "        self.actor_evaluate_net = self.build_net(\n",
    "                input_size=state_dim, hidden_sizes=[32, 64],\n",
    "                output_size=self.action_dim, output_activation=nn.tanh,\n",
    "                learning_rate=0.0001)\n",
    "        self.actor_target_net = models.clone_model(self.actor_evaluate_net)\n",
    "        self.actor_target_net.set_weights(self.actor_evaluate_net.get_weights())\n",
    "\n",
    "        self.critic_evaluate_net = self.build_net(\n",
    "                input_size=state_dim+self.action_dim, hidden_sizes=[64, 128],\n",
    "                learning_rate=0.001)\n",
    "        self.critic_target_net = models.clone_model(self.critic_evaluate_net)\n",
    "        self.critic_target_net.set_weights(self.critic_evaluate_net.get_weights())\n",
    "\n",
    "    def build_net(self, input_size=None, hidden_sizes=None, output_size=1,\n",
    "                activation=nn.relu, output_activation=None,\n",
    "                loss=losses.mse, learning_rate=0.001):\n",
    "        model = keras.Sequential()\n",
    "        for layer, hidden_size in enumerate(hidden_sizes):\n",
    "            kwargs = {'input_shape' : (input_size,)} if layer == 0 else {}\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation, **kwargs))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.noise = OrnsteinUhlenbeckProcess(np.zeros((self.action_dim,)))\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train' and self.replayer.count < 3000:\n",
    "            action = np.random.uniform(self.action_low, self.action_high)\n",
    "        else:\n",
    "            action = self.actor_evaluate_net.predict(observation[np.newaxis])[0]\n",
    "        if self.mode == 'train':\n",
    "            noise = self.noise(sigma=0.1)\n",
    "            action = (action + noise).clip(self.action_low, self.action_high)\n",
    "\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                state, _, _, act, next_state, reward, done, _ = self.trajectory[-8:]\n",
    "                self.replayer.store(state, act, reward, next_state, done)\n",
    "\n",
    "            if self.replayer.count >= 3000:\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def update_net(self, target_net, evaluate_net, learning_rate=0.005):\n",
    "        average_weights = [(1. - learning_rate) * t + learning_rate * e for t, e\n",
    "                in zip(target_net.get_weights(), evaluate_net.get_weights())]\n",
    "        target_net.set_weights(average_weights)\n",
    "\n",
    "    def learn(self):\n",
    "        # replay\n",
    "        states, actions, rewards, next_states, dones = self.replayer.sample(64)\n",
    "        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)\n",
    "\n",
    "        # learn critic\n",
    "        next_actions = self.actor_target_net.predict(next_states)\n",
    "        next_noises = np.random.normal(0, 0.2, size=next_actions.shape)\n",
    "        next_actions = (next_actions + next_noises).clip(self.action_low, self.action_high)\n",
    "        state_actions = np.hstack([states, actions])\n",
    "        next_state_actions = np.hstack([next_states, next_actions])\n",
    "        next_qs = self.critic_target_net.predict(next_state_actions)[:, 0]\n",
    "        targets = rewards + (1. - dones) * self.gamma * next_qs\n",
    "        self.critic_evaluate_net.fit(state_actions, targets[:, np.newaxis], verbose=0)\n",
    "\n",
    "        # learn actor\n",
    "        with tf.GradientTape() as tape:\n",
    "            action_tensor = self.actor_evaluate_net(state_tensor)\n",
    "            state_action_tensor = tf.concat([state_tensor, action_tensor], axis=1)\n",
    "            q_tensor = self.critic_evaluate_net(state_action_tensor)\n",
    "            loss_tensor = -tf.reduce_mean(q_tensor)\n",
    "        grad_tensors = tape.gradient(loss_tensor, self.actor_evaluate_net.variables)\n",
    "        self.actor_evaluate_net.optimizer.apply_gradients(zip(\n",
    "                grad_tensors, self.actor_evaluate_net.variables))\n",
    "\n",
    "        self.update_net(self.critic_target_net, self.critic_evaluate_net)\n",
    "        self.update_net(self.actor_target_net, self.actor_evaluate_net)\n",
    "\n",
    "\n",
    "agent = DDPGAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:22:28 [INFO] ==== train ====\n",
      "22:22:28 [DEBUG] train episode 0: reward = -1744.13, steps = 200\n",
      "22:22:28 [DEBUG] train episode 1: reward = -1025.25, steps = 200\n",
      "22:22:28 [DEBUG] train episode 2: reward = -1590.20, steps = 200\n",
      "22:22:28 [DEBUG] train episode 3: reward = -1137.77, steps = 200\n",
      "22:22:28 [DEBUG] train episode 4: reward = -1675.82, steps = 200\n",
      "22:22:28 [DEBUG] train episode 5: reward = -1632.97, steps = 200\n",
      "22:22:29 [DEBUG] train episode 6: reward = -753.85, steps = 200\n",
      "22:22:29 [DEBUG] train episode 7: reward = -1833.66, steps = 200\n",
      "22:22:29 [DEBUG] train episode 8: reward = -936.49, steps = 200\n",
      "22:22:29 [DEBUG] train episode 9: reward = -1622.68, steps = 200\n",
      "22:22:29 [DEBUG] train episode 10: reward = -1307.43, steps = 200\n",
      "22:22:29 [DEBUG] train episode 11: reward = -908.99, steps = 200\n",
      "22:22:29 [DEBUG] train episode 12: reward = -1504.19, steps = 200\n",
      "22:22:29 [DEBUG] train episode 13: reward = -1003.41, steps = 200\n",
      "22:22:29 [DEBUG] train episode 14: reward = -921.67, steps = 200\n",
      "22:23:21 [DEBUG] train episode 15: reward = -1026.82, steps = 200\n",
      "22:24:34 [DEBUG] train episode 16: reward = -1373.11, steps = 200\n",
      "22:25:47 [DEBUG] train episode 17: reward = -1548.92, steps = 200\n",
      "22:26:42 [DEBUG] train episode 18: reward = -1734.48, steps = 200\n",
      "22:27:39 [DEBUG] train episode 19: reward = -1429.13, steps = 200\n",
      "22:28:37 [DEBUG] train episode 20: reward = -1562.60, steps = 200\n",
      "22:29:35 [DEBUG] train episode 21: reward = -1757.53, steps = 200\n",
      "22:30:29 [DEBUG] train episode 22: reward = -1538.83, steps = 200\n",
      "22:31:34 [DEBUG] train episode 23: reward = -1377.11, steps = 200\n",
      "22:32:34 [DEBUG] train episode 24: reward = -1086.61, steps = 200\n",
      "22:33:34 [DEBUG] train episode 25: reward = -1243.76, steps = 200\n",
      "22:34:32 [DEBUG] train episode 26: reward = -1161.08, steps = 200\n",
      "22:35:34 [DEBUG] train episode 27: reward = -980.70, steps = 200\n",
      "22:36:45 [DEBUG] train episode 28: reward = -872.02, steps = 200\n",
      "22:38:03 [DEBUG] train episode 29: reward = -1005.65, steps = 200\n",
      "22:39:21 [DEBUG] train episode 30: reward = -849.75, steps = 200\n",
      "22:40:51 [DEBUG] train episode 31: reward = -969.59, steps = 200\n",
      "22:42:26 [DEBUG] train episode 32: reward = -382.24, steps = 200\n",
      "22:44:05 [DEBUG] train episode 33: reward = -748.97, steps = 200\n",
      "22:45:40 [DEBUG] train episode 34: reward = -614.85, steps = 200\n",
      "22:47:14 [DEBUG] train episode 35: reward = -507.74, steps = 200\n",
      "22:48:35 [DEBUG] train episode 36: reward = -750.61, steps = 200\n",
      "22:50:04 [DEBUG] train episode 37: reward = -127.51, steps = 200\n",
      "22:51:33 [DEBUG] train episode 38: reward = -125.73, steps = 200\n",
      "22:53:01 [DEBUG] train episode 39: reward = -369.50, steps = 200\n",
      "22:54:35 [DEBUG] train episode 40: reward = -535.08, steps = 200\n",
      "22:56:05 [DEBUG] train episode 41: reward = -623.69, steps = 200\n",
      "22:57:33 [DEBUG] train episode 42: reward = -124.19, steps = 200\n",
      "22:59:02 [DEBUG] train episode 43: reward = -1.74, steps = 200\n",
      "23:00:38 [DEBUG] train episode 44: reward = -126.13, steps = 200\n",
      "23:02:20 [DEBUG] train episode 45: reward = -121.12, steps = 200\n",
      "23:03:58 [DEBUG] train episode 46: reward = -238.68, steps = 200\n",
      "23:05:33 [DEBUG] train episode 47: reward = -121.55, steps = 200\n",
      "23:07:07 [DEBUG] train episode 48: reward = -128.42, steps = 200\n",
      "23:08:39 [DEBUG] train episode 49: reward = -126.94, steps = 200\n",
      "23:10:14 [DEBUG] train episode 50: reward = -488.97, steps = 200\n",
      "23:12:01 [DEBUG] train episode 51: reward = -123.99, steps = 200\n",
      "23:14:08 [DEBUG] train episode 52: reward = -240.86, steps = 200\n",
      "23:16:12 [DEBUG] train episode 53: reward = -248.66, steps = 200\n",
      "23:18:12 [DEBUG] train episode 54: reward = -248.74, steps = 200\n",
      "23:20:13 [DEBUG] train episode 55: reward = -490.37, steps = 200\n",
      "23:22:14 [DEBUG] train episode 56: reward = -125.50, steps = 200\n",
      "23:24:19 [DEBUG] train episode 57: reward = -371.78, steps = 200\n",
      "23:26:22 [DEBUG] train episode 58: reward = -121.67, steps = 200\n",
      "23:28:23 [DEBUG] train episode 59: reward = -239.14, steps = 200\n",
      "23:30:26 [DEBUG] train episode 60: reward = -125.34, steps = 200\n",
      "23:32:29 [DEBUG] train episode 61: reward = -245.64, steps = 200\n",
      "23:34:29 [DEBUG] train episode 62: reward = -239.81, steps = 200\n",
      "23:36:31 [DEBUG] train episode 63: reward = -121.81, steps = 200\n",
      "23:38:37 [DEBUG] train episode 64: reward = -240.38, steps = 200\n",
      "23:40:41 [DEBUG] train episode 65: reward = -122.90, steps = 200\n",
      "23:42:51 [DEBUG] train episode 66: reward = -127.64, steps = 200\n",
      "23:45:05 [DEBUG] train episode 67: reward = -368.60, steps = 200\n",
      "23:47:19 [DEBUG] train episode 68: reward = -480.96, steps = 200\n",
      "23:49:36 [DEBUG] train episode 69: reward = -240.60, steps = 200\n",
      "23:51:50 [DEBUG] train episode 70: reward = -121.73, steps = 200\n",
      "23:54:04 [DEBUG] train episode 71: reward = -125.15, steps = 200\n",
      "23:56:06 [DEBUG] train episode 72: reward = -474.45, steps = 200\n",
      "23:58:08 [DEBUG] train episode 73: reward = -127.03, steps = 200\n",
      "00:00:09 [DEBUG] train episode 74: reward = -234.61, steps = 200\n",
      "00:02:11 [DEBUG] train episode 75: reward = -125.06, steps = 200\n",
      "00:04:12 [DEBUG] train episode 76: reward = -360.36, steps = 200\n",
      "00:06:13 [DEBUG] train episode 77: reward = -123.72, steps = 200\n",
      "00:08:12 [DEBUG] train episode 78: reward = -120.88, steps = 200\n",
      "00:10:13 [DEBUG] train episode 79: reward = -124.85, steps = 200\n",
      "00:12:12 [DEBUG] train episode 80: reward = -510.35, steps = 200\n",
      "00:14:11 [DEBUG] train episode 81: reward = -252.65, steps = 200\n",
      "00:16:11 [DEBUG] train episode 82: reward = -598.00, steps = 200\n",
      "00:18:11 [DEBUG] train episode 83: reward = -125.84, steps = 200\n",
      "00:20:11 [DEBUG] train episode 84: reward = -122.68, steps = 200\n",
      "00:22:12 [DEBUG] train episode 85: reward = -122.48, steps = 200\n",
      "00:24:12 [DEBUG] train episode 86: reward = -596.71, steps = 200\n",
      "00:26:14 [DEBUG] train episode 87: reward = -242.67, steps = 200\n",
      "00:28:17 [DEBUG] train episode 88: reward = -124.79, steps = 200\n",
      "00:30:17 [DEBUG] train episode 89: reward = -118.99, steps = 200\n",
      "00:32:22 [DEBUG] train episode 90: reward = -122.25, steps = 200\n",
      "00:34:22 [DEBUG] train episode 91: reward = -119.55, steps = 200\n",
      "00:36:21 [DEBUG] train episode 92: reward = -237.18, steps = 200\n",
      "00:38:22 [DEBUG] train episode 93: reward = -124.58, steps = 200\n",
      "00:40:25 [DEBUG] train episode 94: reward = -124.54, steps = 200\n",
      "00:42:26 [DEBUG] train episode 95: reward = -516.10, steps = 200\n",
      "00:44:26 [DEBUG] train episode 96: reward = -122.73, steps = 200\n",
      "00:46:25 [DEBUG] train episode 97: reward = -362.34, steps = 200\n",
      "00:48:24 [DEBUG] train episode 98: reward = -120.02, steps = 200\n",
      "00:50:23 [DEBUG] train episode 99: reward = -470.78, steps = 200\n",
      "00:52:23 [DEBUG] train episode 100: reward = -240.18, steps = 200\n",
      "00:54:21 [DEBUG] train episode 101: reward = -235.94, steps = 200\n",
      "00:56:21 [DEBUG] train episode 102: reward = -496.23, steps = 200\n",
      "00:58:22 [DEBUG] train episode 103: reward = -491.39, steps = 200\n",
      "01:00:22 [DEBUG] train episode 104: reward = -494.63, steps = 200\n",
      "01:02:20 [DEBUG] train episode 105: reward = -122.63, steps = 200\n",
      "01:04:19 [DEBUG] train episode 106: reward = -243.02, steps = 200\n",
      "01:06:17 [DEBUG] train episode 107: reward = -355.75, steps = 200\n",
      "01:08:16 [DEBUG] train episode 108: reward = -118.91, steps = 200\n",
      "01:10:15 [DEBUG] train episode 109: reward = -486.70, steps = 200\n",
      "01:12:14 [DEBUG] train episode 110: reward = -124.55, steps = 200\n",
      "01:14:13 [DEBUG] train episode 111: reward = -124.19, steps = 200\n",
      "01:16:14 [DEBUG] train episode 112: reward = -353.29, steps = 200\n",
      "01:18:12 [DEBUG] train episode 113: reward = -119.65, steps = 200\n",
      "01:20:10 [DEBUG] train episode 114: reward = -117.90, steps = 200\n",
      "01:22:09 [DEBUG] train episode 115: reward = -468.36, steps = 200\n",
      "01:24:09 [DEBUG] train episode 116: reward = -369.00, steps = 200\n",
      "01:26:09 [DEBUG] train episode 117: reward = -123.22, steps = 200\n",
      "01:28:09 [DEBUG] train episode 118: reward = -237.21, steps = 200\n",
      "01:30:10 [DEBUG] train episode 119: reward = -124.54, steps = 200\n",
      "01:32:10 [DEBUG] train episode 120: reward = -363.51, steps = 200\n",
      "01:34:10 [DEBUG] train episode 121: reward = -119.92, steps = 200\n",
      "01:36:10 [DEBUG] train episode 122: reward = -241.55, steps = 200\n",
      "01:38:12 [DEBUG] train episode 123: reward = -501.51, steps = 200\n",
      "01:40:12 [DEBUG] train episode 124: reward = -121.89, steps = 200\n",
      "01:42:11 [DEBUG] train episode 125: reward = -360.58, steps = 200\n",
      "01:44:11 [DEBUG] train episode 126: reward = -124.94, steps = 200\n",
      "01:46:11 [DEBUG] train episode 127: reward = -124.46, steps = 200\n",
      "01:48:12 [DEBUG] train episode 128: reward = -120.90, steps = 200\n",
      "01:50:12 [DEBUG] train episode 129: reward = -350.52, steps = 200\n",
      "01:52:12 [DEBUG] train episode 130: reward = -1.16, steps = 200\n",
      "01:54:14 [DEBUG] train episode 131: reward = -359.38, steps = 200\n",
      "01:56:13 [DEBUG] train episode 132: reward = -123.24, steps = 200\n",
      "01:58:12 [DEBUG] train episode 133: reward = -118.54, steps = 200\n",
      "02:00:11 [DEBUG] train episode 134: reward = -493.59, steps = 200\n",
      "02:02:11 [DEBUG] train episode 135: reward = -520.58, steps = 200\n",
      "02:04:09 [DEBUG] train episode 136: reward = -240.74, steps = 200\n",
      "02:06:09 [DEBUG] train episode 137: reward = -476.74, steps = 200\n",
      "02:08:10 [DEBUG] train episode 138: reward = -240.46, steps = 200\n",
      "02:10:09 [DEBUG] train episode 139: reward = -358.08, steps = 200\n",
      "02:12:09 [DEBUG] train episode 140: reward = -362.48, steps = 200\n",
      "02:14:09 [DEBUG] train episode 141: reward = -354.69, steps = 200\n",
      "02:16:08 [DEBUG] train episode 142: reward = -519.00, steps = 200\n",
      "02:18:08 [DEBUG] train episode 143: reward = -120.05, steps = 200\n",
      "02:20:07 [DEBUG] train episode 144: reward = -119.95, steps = 200\n",
      "02:22:07 [DEBUG] train episode 145: reward = -121.48, steps = 200\n",
      "02:24:06 [DEBUG] train episode 146: reward = -125.48, steps = 200\n",
      "02:26:06 [DEBUG] train episode 147: reward = -243.18, steps = 200\n",
      "02:28:07 [DEBUG] train episode 148: reward = -251.35, steps = 200\n",
      "02:30:05 [DEBUG] train episode 149: reward = -594.13, steps = 200\n",
      "02:32:03 [DEBUG] train episode 150: reward = -614.17, steps = 200\n",
      "02:34:01 [DEBUG] train episode 151: reward = -360.23, steps = 200\n",
      "02:35:59 [DEBUG] train episode 152: reward = -2.58, steps = 200\n",
      "02:37:59 [DEBUG] train episode 153: reward = -472.12, steps = 200\n",
      "02:40:02 [DEBUG] train episode 154: reward = -495.22, steps = 200\n",
      "02:42:01 [DEBUG] train episode 155: reward = -492.00, steps = 200\n",
      "02:43:59 [DEBUG] train episode 156: reward = -239.79, steps = 200\n",
      "02:45:57 [DEBUG] train episode 157: reward = -124.08, steps = 200\n",
      "02:47:56 [DEBUG] train episode 158: reward = -236.19, steps = 200\n",
      "02:49:54 [DEBUG] train episode 159: reward = -124.14, steps = 200\n",
      "02:51:52 [DEBUG] train episode 160: reward = -351.57, steps = 200\n",
      "02:53:51 [DEBUG] train episode 161: reward = -243.86, steps = 200\n",
      "02:55:49 [DEBUG] train episode 162: reward = -592.03, steps = 200\n",
      "02:57:39 [DEBUG] train episode 163: reward = -246.36, steps = 200\n",
      "02:59:23 [DEBUG] train episode 164: reward = -122.62, steps = 200\n",
      "03:01:08 [DEBUG] train episode 165: reward = -124.00, steps = 200\n",
      "03:02:53 [DEBUG] train episode 166: reward = -244.09, steps = 200\n",
      "03:04:39 [DEBUG] train episode 167: reward = -599.97, steps = 200\n",
      "03:06:24 [DEBUG] train episode 168: reward = -243.81, steps = 200\n",
      "03:08:11 [DEBUG] train episode 169: reward = -247.69, steps = 200\n",
      "03:09:57 [DEBUG] train episode 170: reward = -588.21, steps = 200\n",
      "03:11:43 [DEBUG] train episode 171: reward = -123.71, steps = 200\n",
      "03:13:29 [DEBUG] train episode 172: reward = -123.24, steps = 200\n",
      "03:15:15 [DEBUG] train episode 173: reward = -560.48, steps = 200\n",
      "03:17:00 [DEBUG] train episode 174: reward = -239.93, steps = 200\n",
      "03:18:46 [DEBUG] train episode 175: reward = -474.06, steps = 200\n",
      "03:20:30 [DEBUG] train episode 176: reward = -125.17, steps = 200\n",
      "03:22:13 [DEBUG] train episode 177: reward = -505.22, steps = 200\n",
      "03:23:56 [DEBUG] train episode 178: reward = -123.46, steps = 200\n",
      "03:25:39 [DEBUG] train episode 179: reward = -354.98, steps = 200\n",
      "03:27:24 [DEBUG] train episode 180: reward = -606.40, steps = 200\n",
      "03:29:07 [DEBUG] train episode 181: reward = -243.73, steps = 200\n",
      "03:30:51 [DEBUG] train episode 182: reward = -237.82, steps = 200\n",
      "03:32:33 [DEBUG] train episode 183: reward = -245.63, steps = 200\n",
      "03:34:16 [DEBUG] train episode 184: reward = -357.65, steps = 200\n",
      "03:35:59 [DEBUG] train episode 185: reward = -124.40, steps = 200\n",
      "03:37:41 [DEBUG] train episode 186: reward = -121.28, steps = 200\n",
      "03:39:24 [DEBUG] train episode 187: reward = -1.86, steps = 200\n",
      "03:41:07 [DEBUG] train episode 188: reward = -356.27, steps = 200\n",
      "03:42:51 [DEBUG] train episode 189: reward = -1.59, steps = 200\n",
      "03:44:35 [DEBUG] train episode 190: reward = -239.10, steps = 200\n",
      "03:46:18 [DEBUG] train episode 191: reward = -471.49, steps = 200\n",
      "03:48:01 [DEBUG] train episode 192: reward = -241.57, steps = 200\n",
      "03:49:43 [DEBUG] train episode 193: reward = -568.13, steps = 200\n",
      "03:51:26 [DEBUG] train episode 194: reward = -125.65, steps = 200\n",
      "03:53:10 [DEBUG] train episode 195: reward = -362.44, steps = 200\n",
      "03:54:53 [DEBUG] train episode 196: reward = -241.77, steps = 200\n",
      "03:56:36 [DEBUG] train episode 197: reward = -481.28, steps = 200\n",
      "03:58:20 [DEBUG] train episode 198: reward = -363.06, steps = 200\n",
      "04:00:02 [DEBUG] train episode 199: reward = -125.28, steps = 200\n",
      "04:01:46 [DEBUG] train episode 200: reward = -492.51, steps = 200\n",
      "04:03:28 [DEBUG] train episode 201: reward = -117.85, steps = 200\n",
      "04:05:09 [DEBUG] train episode 202: reward = -124.04, steps = 200\n",
      "04:06:51 [DEBUG] train episode 203: reward = -120.50, steps = 200\n",
      "04:08:34 [DEBUG] train episode 204: reward = -246.64, steps = 200\n",
      "04:10:26 [DEBUG] train episode 205: reward = -124.52, steps = 200\n",
      "04:12:09 [DEBUG] train episode 206: reward = -124.92, steps = 200\n",
      "04:13:51 [DEBUG] train episode 207: reward = -119.11, steps = 200\n",
      "04:15:33 [DEBUG] train episode 208: reward = -118.05, steps = 200\n",
      "04:17:15 [DEBUG] train episode 209: reward = -126.10, steps = 200\n",
      "04:18:56 [DEBUG] train episode 210: reward = -121.87, steps = 200\n",
      "04:18:56 [INFO] ==== test ====\n",
      "04:19:19 [DEBUG] test episode 0: reward = -117.49, steps = 200\n",
      "04:19:41 [DEBUG] test episode 1: reward = -120.82, steps = 200\n",
      "04:20:03 [DEBUG] test episode 2: reward = -0.43, steps = 200\n",
      "04:20:27 [DEBUG] test episode 3: reward = -609.62, steps = 200\n",
      "04:20:49 [DEBUG] test episode 4: reward = -125.38, steps = 200\n",
      "04:21:12 [DEBUG] test episode 5: reward = -126.21, steps = 200\n",
      "04:21:35 [DEBUG] test episode 6: reward = -119.12, steps = 200\n",
      "04:21:57 [DEBUG] test episode 7: reward = -489.59, steps = 200\n",
      "04:22:20 [DEBUG] test episode 8: reward = -497.19, steps = 200\n",
      "04:22:43 [DEBUG] test episode 9: reward = -119.96, steps = 200\n",
      "04:23:05 [DEBUG] test episode 10: reward = -121.93, steps = 200\n",
      "04:23:27 [DEBUG] test episode 11: reward = -0.18, steps = 200\n",
      "04:23:50 [DEBUG] test episode 12: reward = -484.38, steps = 200\n",
      "04:24:12 [DEBUG] test episode 13: reward = -357.51, steps = 200\n",
      "04:24:34 [DEBUG] test episode 14: reward = -125.31, steps = 200\n",
      "04:24:57 [DEBUG] test episode 15: reward = -636.04, steps = 200\n",
      "04:25:19 [DEBUG] test episode 16: reward = -241.37, steps = 200\n",
      "04:25:42 [DEBUG] test episode 17: reward = -496.39, steps = 200\n",
      "04:26:04 [DEBUG] test episode 18: reward = -125.82, steps = 200\n",
      "04:26:27 [DEBUG] test episode 19: reward = -233.57, steps = 200\n",
      "04:26:51 [DEBUG] test episode 20: reward = -362.01, steps = 200\n",
      "04:27:13 [DEBUG] test episode 21: reward = -123.02, steps = 200\n",
      "04:27:36 [DEBUG] test episode 22: reward = -125.20, steps = 200\n",
      "04:27:59 [DEBUG] test episode 23: reward = -366.01, steps = 200\n",
      "04:28:21 [DEBUG] test episode 24: reward = -588.12, steps = 200\n",
      "04:28:44 [DEBUG] test episode 25: reward = -119.44, steps = 200\n",
      "04:29:07 [DEBUG] test episode 26: reward = -124.77, steps = 200\n",
      "04:29:29 [DEBUG] test episode 27: reward = -237.18, steps = 200\n",
      "04:29:52 [DEBUG] test episode 28: reward = -233.96, steps = 200\n",
      "04:30:15 [DEBUG] test episode 29: reward = -121.85, steps = 200\n",
      "04:30:37 [DEBUG] test episode 30: reward = -560.96, steps = 200\n",
      "04:30:59 [DEBUG] test episode 31: reward = -124.91, steps = 200\n",
      "04:31:22 [DEBUG] test episode 32: reward = -240.69, steps = 200\n",
      "04:31:44 [DEBUG] test episode 33: reward = -634.98, steps = 200\n",
      "04:32:05 [DEBUG] test episode 34: reward = -126.30, steps = 200\n",
      "04:32:27 [DEBUG] test episode 35: reward = -495.60, steps = 200\n",
      "04:32:49 [DEBUG] test episode 36: reward = -474.51, steps = 200\n",
      "04:33:10 [DEBUG] test episode 37: reward = -551.25, steps = 200\n",
      "04:33:32 [DEBUG] test episode 38: reward = -123.75, steps = 200\n",
      "04:33:53 [DEBUG] test episode 39: reward = -121.79, steps = 200\n",
      "04:34:15 [DEBUG] test episode 40: reward = -357.20, steps = 200\n",
      "04:34:37 [DEBUG] test episode 41: reward = -124.20, steps = 200\n",
      "04:34:58 [DEBUG] test episode 42: reward = -500.24, steps = 200\n",
      "04:35:20 [DEBUG] test episode 43: reward = -122.91, steps = 200\n",
      "04:35:41 [DEBUG] test episode 44: reward = -351.98, steps = 200\n",
      "04:36:02 [DEBUG] test episode 45: reward = -235.28, steps = 200\n",
      "04:36:24 [DEBUG] test episode 46: reward = -123.27, steps = 200\n",
      "04:36:46 [DEBUG] test episode 47: reward = -121.95, steps = 200\n",
      "04:37:06 [DEBUG] test episode 48: reward = -357.05, steps = 200\n",
      "04:37:27 [DEBUG] test episode 49: reward = -121.16, steps = 200\n",
      "04:37:48 [DEBUG] test episode 50: reward = -481.52, steps = 200\n",
      "04:38:08 [DEBUG] test episode 51: reward = -123.90, steps = 200\n",
      "04:38:29 [DEBUG] test episode 52: reward = -0.19, steps = 200\n",
      "04:38:50 [DEBUG] test episode 53: reward = -241.87, steps = 200\n",
      "04:39:11 [DEBUG] test episode 54: reward = -356.29, steps = 200\n",
      "04:39:31 [DEBUG] test episode 55: reward = -355.98, steps = 200\n",
      "04:39:52 [DEBUG] test episode 56: reward = -119.89, steps = 200\n",
      "04:40:12 [DEBUG] test episode 57: reward = -123.31, steps = 200\n",
      "04:40:33 [DEBUG] test episode 58: reward = -123.78, steps = 200\n",
      "04:40:54 [DEBUG] test episode 59: reward = -589.42, steps = 200\n",
      "04:41:15 [DEBUG] test episode 60: reward = -0.22, steps = 200\n",
      "04:41:35 [DEBUG] test episode 61: reward = -123.29, steps = 200\n",
      "04:41:56 [DEBUG] test episode 62: reward = -117.87, steps = 200\n",
      "04:42:17 [DEBUG] test episode 63: reward = -362.99, steps = 200\n",
      "04:42:38 [DEBUG] test episode 64: reward = -123.37, steps = 200\n",
      "04:42:58 [DEBUG] test episode 65: reward = -234.81, steps = 200\n",
      "04:43:19 [DEBUG] test episode 66: reward = -243.27, steps = 200\n",
      "04:43:40 [DEBUG] test episode 67: reward = -620.50, steps = 200\n",
      "04:44:01 [DEBUG] test episode 68: reward = -119.09, steps = 200\n",
      "04:44:21 [DEBUG] test episode 69: reward = -122.85, steps = 200\n",
      "04:44:42 [DEBUG] test episode 70: reward = -245.37, steps = 200\n",
      "04:45:04 [DEBUG] test episode 71: reward = -123.43, steps = 200\n",
      "04:45:24 [DEBUG] test episode 72: reward = -234.45, steps = 200\n",
      "04:45:45 [DEBUG] test episode 73: reward = -244.40, steps = 200\n",
      "04:46:06 [DEBUG] test episode 74: reward = -125.58, steps = 200\n",
      "04:46:27 [DEBUG] test episode 75: reward = -118.40, steps = 200\n",
      "04:46:47 [DEBUG] test episode 76: reward = -616.93, steps = 200\n",
      "04:47:08 [DEBUG] test episode 77: reward = -124.10, steps = 200\n",
      "04:47:29 [DEBUG] test episode 78: reward = -125.62, steps = 200\n",
      "04:47:50 [DEBUG] test episode 79: reward = -119.06, steps = 200\n",
      "04:48:10 [DEBUG] test episode 80: reward = -349.48, steps = 200\n",
      "04:48:31 [DEBUG] test episode 81: reward = -231.84, steps = 200\n",
      "04:48:52 [DEBUG] test episode 82: reward = -0.43, steps = 200\n",
      "04:49:13 [DEBUG] test episode 83: reward = -124.12, steps = 200\n",
      "04:49:34 [DEBUG] test episode 84: reward = -233.98, steps = 200\n",
      "04:49:55 [DEBUG] test episode 85: reward = -0.10, steps = 200\n",
      "04:50:15 [DEBUG] test episode 86: reward = -361.32, steps = 200\n",
      "04:50:35 [DEBUG] test episode 87: reward = -360.94, steps = 200\n",
      "04:50:54 [DEBUG] test episode 88: reward = -236.13, steps = 200\n",
      "04:51:14 [DEBUG] test episode 89: reward = -121.61, steps = 200\n",
      "04:51:34 [DEBUG] test episode 90: reward = -352.96, steps = 200\n",
      "04:51:54 [DEBUG] test episode 91: reward = -235.01, steps = 200\n",
      "04:52:14 [DEBUG] test episode 92: reward = -123.05, steps = 200\n",
      "04:52:34 [DEBUG] test episode 93: reward = -495.73, steps = 200\n",
      "04:52:54 [DEBUG] test episode 94: reward = -244.08, steps = 200\n",
      "04:53:13 [DEBUG] test episode 95: reward = -123.29, steps = 200\n",
      "04:53:33 [DEBUG] test episode 96: reward = -0.95, steps = 200\n",
      "04:53:53 [DEBUG] test episode 97: reward = -242.85, steps = 200\n",
      "04:54:13 [DEBUG] test episode 98: reward = -124.81, steps = 200\n",
      "04:54:33 [DEBUG] test episode 99: reward = -122.80, steps = 200\n",
      "04:54:33 [INFO] average episode reward = -241.67 ± 171.27\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABZZklEQVR4nO29eZQkV3Xn/70RGbnUXl1V3V29qdXqbu0LUiMkZARIAgReBFgCYYywjS1g0IxtsI/FADPMz4NtPGaYYxuDhTEID6uFBRoDBrNjkAStfUetllpdXdVbdW1ZucT2fn/EexEvIiOzsiqrurI67+ecOpUZ64vIiHffXR8JIcAwDMMwCmO1G8AwDMO0FywYGIZhmBgsGBiGYZgYLBgYhmGYGCwYGIZhmBiZ1W5AqwwPD4vt27evdjMYhmHWFPfdd99xIcRI2ro1Lxi2b9+OvXv3rnYzGIZh1hREdKDeOjYlMQzDMDFYMDAMwzAxWDAwDMMwMVgwMAzDMDFYMDAMwzAx2k4wENG1RPQUEe0joltXuz0MwzCdRlsJBiIyAXwMwKsBnAPgTUR0zuq2imEYprNoK8EA4FIA+4QQ+4UQNoAvArhuldt0yvHvjx9BxfFWuxnMSeabj0xgslhdtfPf//wUPH/tl/kvVl38+OljAICjsxX8dN/xVW7R8tNugmEzgIPa9zG5LAYR3UxEe4lo77Fjx05a404F9h8r4vc+uxf/76Hx1W4KcxKZr7p45+fuxx33ja3K+Q+eKOH1f/dTfPeJI6ty/uXkSz8/iJv+8WeYKTu4/e7n8Nuf+TlOtXlt2k0wUMqymjsuhLhNCLFHCLFnZCQ1o5upw6Hpcuw/0xmU7EBDnLdXR1OcKtkAgJmysyrnX04OnihBiEDYzlVcVF0fFcdf7WYtK+0mGMYAbNW+bwHAQ9tlZGKmAgA4LP8znYEyHa6WCVF1nLa39jvQiZlgUFV2vPB+zlXWvsDTaTfB8HMAu4jodCLKArgRwF2r3KZTiolpKRhmWTB0EuVVFgzq/NVTYGStBldl20NZXs9sxV3NJi07bVVETwjhEtEtAL4FwATwj0KIx1a5WacUh2eD0Q5rDJ1FWZqQyqtkSlIC6VTQGMbl4KrqeuH9LFZZMKwoQohvAPjGarfjVEWNdiZYMHQUasReXjVTkhQM7toWDFXXw3EZ2VW2fTYlMacGSlOYKTurNnpkTj6RKWl1Omb1rFXdtf3MHZ2Nwn11H0PxFDMlsWDoMCZmKujNB4oi+xk6h+qqO59PDY1hXIvmKzteKHDnWDAwa5WS7WKm7OCirQMAougK5tRn9Z3PMippjQsG3QRb0QTDLJuSmLWKMiO9QAqGI6wxdAxlO+iQV9vHUF3jgmFcG0xVHA+VU9T5zIKhg1CC4aJtAwDYAd1JsPN5eZiYriCbCbrNsu2hIq+HTUnMmkUJgh3DPegvWByy2kFUVjmPIMxjWOPhqhMzZZw+1A0gcOQrp/qpFpXUduGqTOsIIeD6ApYZl/vK2byxP4+NfXk8eHAaH/v+Prztl06HLwTuuG8M1567Eev78uE+FcfDdMnBxv48GjFTdvD9J4/itS/YDCEEPF8gI88vhMBXHzyEa88dRd4y8INfHMP5m/sx3JOrOc7XH56AgMAv7RzGQFc2ts52fRABlmnguePz+N6TR3FkroLLdgzhJTuHw/PpbbrvwAmMT1dQsEy89gWbYRpB1ZUjsxV85qfP4eGxaZy3qR9TJRvre/P4o1ediYrjYd/RIgpZEzuGu+H5Avc+ewJX7BzG85MlfHnvQbznlbtBlFbBJcD3BX709DG8dPdIuN2ff+MJXLZjCC8/az0Oz1QwPlPGaH8eo/0F3PXQOD714/2Yq7i4/XcuxdZ1XQCAZ4/Po79gYV13FtMlG925DEwifPhbT+LNl56GbUPBdo+Pz8IXAudt7sd81cXzJ0o4OlfF/Qem8JrzR6M8BseDEAL/9uhh/MsDh3Dupj78wTW7G/62Scq2hw987VFMlxx050xsHezCFTuH8cLtgzW/gWIhwXR0roLenIVC1sSR2QoGu7LhyFznF0fm8Pj4LLYPd4e+sjRmSg5+/twJEAFX7h6JvQtCCNiej1zGRMXxMDZVxs71PQ2vWQiBnz83haeOzGHX+l48OzmPkuOGAm8ppiTH85ExCCXbw2/8w704f3Mfzh7tw/5j83j3K3ajK2ti39EibM/Htx87gu89eRTTZRtXnbke11+yFT96+hiuOXsDztzYu+hzLwQLhlOM//a1R/HlvQeRMQz85Nar0F+wwnVHZivoL1jIWyY2DeTx/aeO4eGxGZy5oRfTZQf/7WuP4c++8QQ29uXRncvgq++6Ap/80X58+qfP4b73X1PTET4+PovX/t1P8P0/ehl+8NRRvO/OR3HxtkH8eN8x/PV3n8ZPb70apkF46sgc/vBLD8G9XmDLYBd++9M/h2kQ3vPK3fhPL9sZHq9se7jlC/dDCGDLYAH/8SdX4dM/eRa/OFLEn7/+fPz+Fx9AxjTwN296AX7vs3vx9NEiTIPw9z/cjz965W7cctWu8FjH5qp4yV9+LxaeuX24C5ectg4A8OZ/uBf7jxWxe0MvPvUfzwIIinL94St24/1ffTQsNveVd74Y0yUbb7t9L77z7ivxg6eO4W+/vw83XX5aTIAmuefZSfzWp3+Or7zz8vCct9/9HOaqLn5p1zCu+sgPULI9bB4o4Ce3XoV/uX8MDx+agRDA00fnQsHw25/+GS7bMYQPve58XP2RH+KdLzsDrzl/FH//w/3oL1jh/fvQNx5H1fFxxztfjLd86l7c//x02Japko2sGZk/xqbKeOfn7gcAHJicX7RgeOLwLO64bwzb1nXBFwL/+vAE/vb7+/BbL96OD/7auan7lBcoiXH9x+/Gq87dgD9+1Vm45iM/xLtfuRu/fcXpNdu9918ewX0HpgAA33n3ldi5Pr1T/KtvP4V/uucAAOCTN+3BK87ZEK77j33HcfNn78Pd770Kdz5wCH/xzSfx8AdfiVzGrHvNP/jFMfz2p38OAPjNF52Gvc+dwKxW9ynNlPSLI3N47cd+gm//4ZXYMtgVW2e7Pi7/8+/i/b9yNs7b1I+HDk7joYPT4foXnb4OpkF42+17AQBEwKXb12Fjfx/+6Z4DuP3u4Np685kVEQxsSjrF+I+nj8N2fRSrLo4mnMsVx0PBCh7+977mbHzodecBCBxqh6YCp9rrXrAFXdkMHhufxXTJwdG5Kk7M2zgxb9ec67nJediujwOT85gtBy/G2HQJj4zN4MhsFcfmgphvdexD02U8f2IeANBfsPCTRLni8ZkyhAA29OVCM9fdz0zi6w8H5bIeHpvBgclg/8l5G2+6dCse+x+vQm8+E55LcXSugorj49ZXn4Xb3nIJAGC+GkXm7DtaxH+5ehf+7Q+uxBN/ei0++GvnwvMFjs5V8PTRIga7AoF6vFgNC78dL9ph6YOF8gFUp/H8iRKAYHRYcXxUbA+lqoeS7SGXMXB0rgIhBOYqbqhBKUexOuezx+dxbK6KyXkbY1PlcHSqmwKLVS9s56HpMl6yaxif/90XYVN/HrNlJ4pKcj1Myt+yL59ZUl6DGvX/xa+fj//4k6vw0H9/Jc7b3IcnJmbr7qM0Fjslj6HieHj+RAm/OFLExEwZc1U3fGaSjE2VcMZIYMp57nip7vmmyw6kclhj5nlusoSyEySqTRZtVF0/fH7rod6lf//DK/H2l56BQtbE1Hx03LSSGM8dn0fJ9nBgsradM2UHk/M2DkyWwt/gL3/9Anz57ZcDCAZxzxwrAgA++sYL8f33vAxfevvl+ORNe/D1//IS/K/rL8Dd770KN12+vWG7lwoLhlMM2/NDE8xcQr21XT9Uz3dv6MWbXrgNlkkYn65gYqaM4Z4c/vz15+O3rtgOQEZdyA4lzVGtJ/cUq8FLMjFdCSu3qnBYFfs9MV3B+HQFRMDZo701SUGqjtOu9b1wfRF0pq6P2YqLqXkbh2crmKu4siN1MNCVRd4y0ZU1azo45eQ8c0MvNsiRvVqmOlQ1irNMA5sHCrKtFRyaKmOXHIkG98CPrlO2eSEnrtpHlU9Q+1XcKMRxfV8OjidQcXzMVRyMKMEg1/u+QLHqBoJ7Ouhc5ipu2NHpv0nV8cJRa7HiYveGXrxYmuPmKpHJQwjgxHwgRId7c0sKX63Izj0vBxk9uQx2DPc0DGZQiW1pUUnjWsVfJRDSRuCO5+PoXBWXnh5oYI3CrW3XQ2/eSj1nlNPRfOayao/SEguWiROlaLBUTNm/Gjqma9cp4a7nQmwaKOCS0wZhGoTDsxVMzFTQk8vgdS/Ygu3D3eG+Z4/24YY9WzHaX2jY5lZgwXCK4XoCA3K0m+x4bc+P2W0NgzDaX8D4dBnjMxVsGogeekB2ivLhTivTrdtX1bnGp8uaYAg6ivHwfxnj02WM9OQwKDssHRUKuGNEOfeicMD7DgSTvMxVHFRdH44n0JMLLKF5yww7q/BaZbuzGQM5ywivX7URADZpfpNRee3PHZ/H8WI1bEPV8cNOrViNBODCgsGLnWtO0zTUuvW9ebnOwVzFxUhvXDDM25FmcPBEObZtcH/joZPFqgvPF5i3vTCJsTefwVzFjQkApV2N9CxNMKiONac9S6MDeRyeqcCvMxFPpDHUCobwOZkuY0zdr2ptZ3psrgohgHM39SNjUENBZLt+eA+qiWtUbQme7+YS1NR6/ZmbkppX3jJS91eCIU2bUMKiqj0PecuAaRDW9+ZweKaKwzOVBX17KwULhlMMx/OxTmoMSYeY7fqhrVkx2p/HhOywN8kRSD4UDFHUxUSKYAhH0lUXRWmmGZ+pRBqCqsukfZ+YqWB0oBB0WNV0jWG7jPrQR1P37J8EoEbMwX598sXPSyeijnopcxkjvGbVKSlBNToQjbjU6Ou+5wP7dSic3EhjmKs44T1dqEMtJwWDEih2dE1KQ5iVmsh6KRiSsfGOJ/CgtD8H97rWlFSWgkF1OGq03Ju3MFuJlz9RZR1GenOh4F8M6t6q5wQANvUXYHs+js+nzxCnOuA0waAGEiXbw+PjgTkqraNVwRObBwrY0JdfQEPx62oM5ZjG0Fy46VzFRU8uEwYv5C0TU6XgXo/05uoIhvrlMkINUtPK1f3c2J/HEakxjLJgYJYDx/Mx2C0FQ+KBrLp+TaTHpoFCYEqaLoej5lBjcL3w4R5vYEqa00xJj4/PhC+bEgjjmoAYnyljU38ePblMrSlJmrOUw7xi++FLfO+zJ8JrUP4O9eLnLaOuKSmbMcJrVtei2qW/dH35DLqzJvY+F5znjJGe8BrD66xGQmmxpqQ5zZQUagx9gSCYKTso2m74Pa3+zt4DQbt07ex40Q6vSZ1PdZ69cmTbJzUGvb3HipFgsF1/0dNtJjsyIHiOgEi4J4lqJdU3JQHAz+TvnDbKVoJwY38emwbysf2SxDSGuoKh+fkU5ipOqC0AwTsyLU1J63vzKDseXC/9GUwTGrPac1QJBW3wnG7sCwZrh2cq2NggwGElYcFwiuF4ItQYkiNyx6sVDKP9eYzPlDFve5rGEEWwqBc67SUMOzBtFPvYeOSAjCq5BvvO2x6enyxh00ABvXmr5mVSI6RCNuhwyk50/sfGZ8LtlMlJvag5q1ZjUGYjXTDoGsO67mysYyMijA4U8IsjgcNvhxQMVccPR7tFTVupLFCAMDQlybYWNVNSUmM4MluBEMBgVxYZg6L6O9rvp4+kdU1Qjf6Tpqu4KckJo4KAyJSknN2LLWyna2MKJWTr2f3DiXoWEAxPHFbXWdtRq+dJhfg2qvVVdX10Z02YBtWcM5y0KKYNLqwxqHsKAIWsCVcKVPU7JjV0dZ+KKWYxXfNMClqlDR2dY42BWSZc36/vY3D92MsMBCM9NV2tGvXlYz6G+oIhrEWvOWXVy9JfsDAxU4bvCxyeqWDrukK4flRqDED8ZZqQcf2hYNJeGn1Qq0al6kUNfAwJB6Nsd9Y0wjBE9aJOzJRDf0ryXgBBaOCWwQJMgwKtSes8dKdhI3Tb9VzFCU1JVSc6ntIQ1L3tyWVQsMzId6P9fur6dVOa2lcIEV6b0lB6QsFgoVh1UZFRUEAgGLqyJrqzkclwMTTSGMbraAyN5mOYmKlgmwzPFdp1Jjk8U0beMtBfsKQJtFJ3rmUVaJHLGDWCT5+bQq1LDqKSFKtxwaCeUQChbyjZ5moDoaMEXyXmY4hMSSXbgy+AjSvoYG4EC4ZTCCEEHE8gJyN1kiMV26v1MegdpDIlqQdUH7GnRiW5tRqD4pLTBjExU8Hx+SocT2CPjOUPzlkIOy79pZmYrmDTQCE6v2aP11Gj0p7Qx2DUOBjVKDFnmWGHqDuf0yI6lDN6Q28elmkgnwlMVDHnc7OmJE2jmJipxPYLNYZeJRiUoLOQz5oxTSxJserElh+ercRMJereRD6GDHwBTM5XsU6aGI8Vq+jJZWIDgMWgBIk+yBjsspDLGHU1hmgGt9pzHZou47zNfbHOdjZlbugJaVohIoz252G7fhh6m6TqeshmTCkY6piS3MVEJTnhPQXiQnF9PcHQQOjoz0NSMOhaAmsMTMuo0bplEHpymZoH1U7xMegdpArZVKacquacOzJbqbGhVhIjaTUCzVsGzhntw9G5ahhNc/Fpg+F+mwYKoQ1cdXLBqNoNTElah6V3wMM9QcemOtI++aKmmpKUj8GMnM/V0PdRiUUkJe/F5sFIc9LDVfVQ0YVG2fr6Q9NlLf9B8zHIqCTd/FOwzJgmppar/xXHx1TJDu/fxEwldu1JbUp1ZpPzdhjGfHS2GpwruzTBUHU9mAbFsomJKPRXJRFChL9jUmMQQoSBD+r5W9+bQ9X1a0xAR2ajKB01kq5X1kVpx7mMWZNtrX6bqrM4U1JPPu5jUEQaQ2Ig1sDHoISF3oa8fDc3aH4FjkpiWsaRL52VMdCTEvVju35NmQzlV8gYFNqc1QNadgJV2zIJvgCOJJLIonBVB8WKi50bgtj/TQMFjA7k4fkCD49NAwAu3NIfRnRs6s+HHVYUehlFCqkOa6bsQAiEprGzNvbJbeM+BjWy16lqzmfDIGQMgu35kQAaqNUYlMakm9R0jWG27GBeC3VsRMWNTDcT0xXNphz5GIZ6siDSfCZSMISdldznTHlfz5IZrodnKtjQn0dvPoOJ6XLs2tWxlOBQAkIIYF13cB/LjoeevBWa2BZvSvLDZ0RH+auS2J4PIQDLJDieiIW0TpUcVBwfmwYK2CzzSs4aDX7nZEcb+KCC30ZpuvUc0Co0O2elmJKW4HyerbhhFBxQTzDU8TGkHDsZvpwxKCwnojucN51qpiQi+l9E9CQRPUxEdxLRgFy+nYjKRPSg/PuEts8lRPQIEe0jor+mRsVomBocL3jhMgahNyXqJ01j6CsE0Tgb+vJhx62PJMu2h9Nk+GgyZFWZBWbLLuZtD7tlvZnNA4XwgVblC7YMdmFDbw6WGQggNfpS5i49t0C9dCr6aPf6eMeohEiP5mNIvvxJB2kuY8B2/ZgDM4lqsxq5qk4lGfEDRHZqIYJs6WT8fsXxsHVdF0yDMD5dDjse3TzXZWXQk81oGlAG+Wytj2GXFAy7N0TX35PLhHb2mMYwE5mlgv9RZzao1Z7qy2divpzFUHE85LSOUbFpoJAalVSRmdz9heD8utYQ/u4D+fC+q99Z72h9X8Q0BiUg6oWsVh2lMdSakkLnsxZYsJDGUKzWNyUpwVDrfK5/bP15qDh+7HjqGguWib7C6lQtWkmN4d8BnCeEuADALwC8V1v3jBDiIvn3Dm35xwHcDGCX/Lt2Bdt3yuFokTg9+UxtHkNKVJKKxtF9DflM5GOouD52yKzL+2WMv0J1KEfngpfzjPU9MCjoYNXDffczk8hlDAx2WRgdCJYb0tQF1NEY5EsyJcMBL9o2AIMQZryOTwdOSKX9NAxXldtkpWCIOqLakZhykKv/QZG1yA6dzBu4d/8krv7ID3Hph76Lux4aT9wbH925DDb05jA+U44JaWVWymcN9OYz4RzCvXkL+YwR08S6siZOHw5G0mePRhpTbz6DjTIyR0/um5ipwDQo7PT1zkwXDLqPQQn4yWIVH/63J+F6Pp45VsRffeupVOdu1U3XGDb153F0rlIjpFX7+gu14aP673H6cBcsk2KC4bHxGThe4EtwPBEK9KHuLCwzSnJ76vAc9h0thu2tymc9KwXDdMnGe778UBChpSe4pZiS/vXhcVz3sZ/gV//mP/Djp4+F5Ux6c/GoJIWeqOj7Av/w4/0o2a6W+exipuTgc/ceCNsXi0pyvZhgyFsmBroCB/tqjY1XTBwJIb6tfb0HwPWNtieiUQB9Qoi75ffPAngtgG+uVBtPNdxQYzDQk8vg+Fy8Rks1JcENAG699qzYg24YhGwmyOb0fIGzRvvw/IkS/uwbT+LLe8cwULDw0TdeFL5Ux4tBB95fsPD+Xz4Hl5w2iE0DQVTPdNnBO166A0SEN+7ZGnb2ffm4j2GyqEIos+ELpTSGczf14f4PvAJd2ahjUaM0IMp8FkLggYPTOGtjL2zPh2USDKkFZWV0igrVXK/trzhtqBt//5ZL8JJdw/K4SmOotY+XHQ/ffvwIDk4F9ziZGV5xPOQzhrS7l9GdjV616ZINokBo9eYtQGlAucDur667WA2Sqt74wm04Y6QnlnjYk8ugO5fBviNzMaFou0FUmupQ+mIaQyQkevOa81l23D946hg+/oNn8GsXbsL3njyKv/3+PvzmZafV2LkrTrwjU5y3uR++AF77sZ/iE795cahpqo5Y5afovgN1rUM9OfzmZafhxWcMh53000fn8J5/fgj/540XhXklqhM2DML63nxYw+iGT/wUsxUXF28bwFfe+eLAxyAj0mzXx30HpvCV+8fw+os3x+pGJU1JEzNl3PL5B3C6rKr7lk/9DJ/9nUuD3ycWlRRcv2VSaOqcrbh4bHwW//PrT2DTQCG8zmLVxVcfPIT/ftdjuHLXCLau69JMSUF2v+54B4LBlQoWWA1Olp7yOwC+pH0/nYgeADAL4P1CiB8D2AxgTNtmTC6rgYhuRqBZYNu2bSvS4LVI6GMwKQxT1EkLVwWAa7TKk4p8xoh14l+75Qr8090H8L0nj+Knz0ziwYPTNXb2nlwGb7o0+j3++R2XhzHnAPCGF26Ntk1EJc1VXRleaIIQdGrTMrM0GEEFL4kyDeijt7xlyhpANm74xN344K+eg6oTF4JKY1Bt7sqmP/qvOnejdg8C53NaUlbFDnIwhrpzmJyv1poRHA8DXVn0FSw8dHA61rlOlRwULBNEFJp6TIPQlTVjzmfl8OwvWLj67A2x6ptKMBSrbs3voJuPdI1BL2Pem7fCzkgJlpIswVHUEvkOTZdrBENaoiQAvOKcDfjEb16M//yFB/B/7zmA9/3yOQAizVIJBl2jUPetJ5dBVzaD8zb3hzkrD48F1Wan5m0U+2S2u2Za6c4FZjfPF5ituDANwv3PT0dmRMsMw1X1uZl1U1I14c9R1/2eV+7GBZsHcOX/+j7ufXay5l4qrTYvz5ExCMWqG07xWbKj56ZYdTEmBxBqvZ7XktQYgKBAYaNqrytNS6YkIvoOET2a8nedts37ALgAPicXTQDYJoR4AYB3A/g8EfUBSNOZUoOUhRC3CSH2CCH2jIyMtHIJpxSRYDBkVJJTsz7thU6jkDVjHXMuY+J3X7IDH3nDhQCChz1pm9ZHVABw8bbBuoW+CpYJg6IXpFhxw87eMgmmQeFoUnf06RE6CiXsjsxW4fkC0yUHthe3g+cyJmwvcvwmR2hpKBNVmqO54nph0lN3LoP5hGAI7MYGNskaQnr45XTJDjsCdR09uQyIKJ7HUHVjAlC/5p58Bt05E/O2lyKgrdR9evNRSQeVMxG0VZ0vGj3rI+iaa6+jMRARrj1vFP2FbHgs/fhpGoMuGBQq2kwl9ZU1h73+LKh7pY6vNCL13AY5LMFAoqRNqKPnMSgtMMxol+vyGROjA3kYBDw5MVdzL9Xzowt4/b6VbTcmAPcdDRIn1fOutrO9oG3J5/GCLQMrUk67WVrSGIQQ1zRaT0RvBfArAK4W0rgmhKgCqMrP9xHRMwB2I9AQtmi7bwEQN9wyDVHOZ8sMbNfFalCJlIjgej58gVRTUhp5LeVf7wTCxLSKWxMGqHdiC0EU+BmK2khNvXiqg5xKOX9v3sLxop3qCJyUdXrKjldTFyprKo2hts5PPXIZM+Z8js5noGx7QSG/fCasU6RTdoIS56qG0IHJUihopkp22MGp6wgjrLRKscVEiKT+uTcXOKo9X4TltqN7FG3XJbN/PV+gKxsIA5WspeerAJHGoCfypTmTlWO3HoWsERNW6vh9SjBoJrlixUXBMkOBpbdfZUGXnaiYom7yzEntqhwKhiyOF21Ml225PgpX1bUwlQypRu9EesJZdB7LNLChL48nD9cKBl1jCNZZmKu4sVIX+vuhMup1DVkxXXJCv167sJJRSdcC+BMAvyaEKGnLR4jIlJ93IHAy7xdCTACYI6LLZDTSTQC+tlLtOxXRTUk9uSCxSY2U9BIRzRDUgnHCz4rubCZ8kcqOF+sguhchGICowBsg7ekJG64SDIUUwdSTMCUBwKT0dZSl+Ue/VuWELDtB+G0ybDcN1ZFXXb8mPLHseJiruujNW6l1n9SoWjm5y44X+kWm5p2w4mtSA9ILAiofg6JPE4Y9+Uy4Tvl41IhZF9BKAANBh53XzptPhKvOV6POUzclJammmD50dHMYEOWPDChTktZhzttujaaZDEzQ81mSGoOepKZqhM3oGoP0E6n3YKpkh7Wh1PM91J1FxfHhpGiUmwYK4T3Q739eCijVHvUMqDbrpiQguo/FqhuWU1f+n5my09RA5WSyklFJfwugF8C/J8JSrwTwMBE9BOAOAO8QQpyQ694J4B8A7APwDNjxvChipqSEc1cvKtcMudiIPV6quycb5EhUHC82PWfPogVDJmZKihUpyxrhRCiFrBHbR/+vt09F91SkxpBLEwy21/ToTCW4VR0Pw72RfX6kJ4eynEMhNCXZ6YJBD4tVNXVmyk6NxqA6nULWCKffDCp6Rp2RsmUDgblI3a9JrShe8t7o3/OWGRvh5kIfQ9AZzmvam/pd0k1JfkNTnG4OAxprDHMVt0bTzJgGujTNQNcKagSDZkpTNcKmpQYVlcSIOnxVWyrYLni+1TMc+B/iGuVmLXotNhiRz5ASEKq8uR6Gast6TTpzFQclx4MQ0e81pZkW24UVEwxCiJ1CiK3JsFQhxFeEEOcKIS4UQlwshPh/2j57hRDnCSHOEELcosxPTHPopqTkqEsJhmZGygBQsIzQRFFIPLQ9WsVOPToo2SEthG5Kmq04sU6wYJlhBxI3JWXCNijUS6pGzmXbq8nZUHkMVdcLX+aFCDOfXS/s1IEggqZiBz6GvrxyAidDNH3kLCPWsejx7jU+Bvm/YAXmIccTNfV5iEirgZQJNbTjYURXrubeBNta4bH1EW4uY4AoCledD53PTqjJ1SuF0sgxmk8KBjshGLSR9HzVTdU09evWcz/03y5vGXJdcLxBmcCnnttcxgyDDsry2lRoNRBpDHrmcrI8hcqCT7ZJmbRU2G5v3opX35V1mIYS85rPaWVV1HkDjaG9co3bqzVMS6iSFRkzinZRHW91kRpD3jLDwm3JZKaeXAZT8zaEQExjWKwpSc+10FVroHZkGJ07mmdAbysQjZzLjleTs5ELOwivRtDVIydzChxPRJ1uLkgIDJzPQdJTb8L57PkCtjQ/DXRF0T8q1FK/pr6kKUmz+ydNSfp2uilJmdBUG/V7kzy2LpCIKDBduSoqKTIlqd8lLbO4upDGkI2XKInyGNKjktI0Td1sozuYY8+FTAZUx1d5GqEpSZXE0JzPquS4uk4gnrmcPI8u2NOikgoxjSFyPitTkirjEl6vplWMyOdBiOZ8XicTFgynEHYsKileYVWta+Q01NFfwGQn0KMlZamXKpcxmtZGFMphB6T7GMK2ZGs1Bt38kEuYksoyDDEZrqrCFpsWDJYZamHDvZFgKGTN0OzQm5PRQZpgUB1fXkasKD+DLkSTCWjqutS1TktbeK39PfIj1NMYkppbn6aNJE1YgR8lXrRP9zHocz4o0sIrdZI+hkZ5DMkaRAr9GirSlJRJ1GdSGp06vor7jzSGqLqq2kaZkmLVUTVTUjmpMagseK18u7pG/X9kStKcz65fqzFU3NDxrAsN1hiYFcMNTUm6xiDD4hKZwAuRrzNiB4LOMZoeMni4F2tGUsdRczgnfQyxTFDNbJEcYevrVaXNiu2h6vmp4aoL2cd19O10M03eMrXJgqQpSXM+J4uiqVIbyaQ8/TpUh6/utbq/jTWGyISmJ1olbfahKSlr1ji981onHkUlBRVcVc2eZKG6BaOSEqakZLiq7pSdt9M1BtVmIoTmouRzGAoGeXyVpzFTjmsMyiwHRM+IngUeNyX54TUAkSkp+Xzr4apqfTFhSrITGoMynSY1FQCdE5XEnHySeQxApC47i4xKSqbo6/TlrdgsYMDiHc+AepmCl9H1RaqqnpNF8BQ9WscYtS+4JmVSqbj1w1XLdWLw09BfVl0AJsNne6TzWbnE9JBHICr4li4Y4hpDPiEYahzJWlSWEiaT81XkM2b4GzQyJRUSAkmfy6Ik/STH5oJ8kN0yjl6vmCqEWFBjyCdNSY4Pg6JnRBcMyQFBss1bB7vCzj/pGypIjU5pa6pI4LSuMchnQy1TEUkDqYIh0hiU4FMaQ/KeqraowUdPzoLni9CHoXwMA11ZGDLLPch4diIfQ0/t89AusGA4hXD8NI1haVFJ+mg5TWNQJpZ13TkQLd6/oI6j4vqBeGevOtVCojPQ5zKO2qpGztKUJF/KZFSSynxOHrMeSQGQzRixxLBgeaAx+CKKvkmaI0ZTNIZkB62be4DIFp7sNEPnc85Cdy4KN81ZZixZTicSAgYKMq9BF7xhVJKtQlSDzk0VRRyfLsPzBX76zHGZG9O4I6sxJUnzXTgvRiLBLd2UZIEomHu7LJMMk89hVFMr6PQHUzWG4JzK76DQy4NEpiQHVRmCrQYj3bkMBrqsGgGdZkoK7lVw7+ZtF44nQoG9UVbDnYv5GLTnocln8mRxskpiMCcBR4s8Uh11MRGV1KwpqdBAY4h34AZ6spklawyAVhE0xZSUJpT0/0DkY1Aj0eSoD4jCVYMaRs07n6P2GOjLZ8I5E6JriMJGi1UXXdlM2NGqyJ3dG3qRMSjmyFSCd8tgATvX9+C8zf2x617IlNSdM2FoBdbylpEaygsAv3bhZnRlM8hlAo1BZVmr80XhqsF/JWBVNdePfX8fPvnj/Xjy8Bze/8tn19ybJMqUpJIrVehucorVqiuTBFOenWvP24ienIkjs1UcmCwFYcYJE6D6rhIxQx+D/K7mYwCigowK3ZQ0nNAYks/7tnVdMQ0DCN6x/oKFoYQpVQ3E1PlyVlAPa1QKhnGtBLsuGJr1/Z0sWDCcQri+ikoKHMF5y4gmBGnBlJR8aJO+gB7ZYS4WdZzkPMVAbWap4vIzhvCGPVtwjqw0mrZNxfFgEKVHJS1RY8hlTFy5awQXbOmHqQnXXi06qFhxsb5X8zHIjuvV523EBX/0Moz05sKkOd2U9J13vzS67mxc+0lqYrs39OKMke6wdn9X1pQlFUy8cPs6vPaiTThXChnFmRt7w/IKv/GibbhEmzSpYAWZwUKImlyMkd4cPvzr5+MLPzsYjnKfm5wP7kcjjSEbRLTZno9cJmhfIRsJhmiegtpyGIqX7h7BS3eP4E/ueDjMY0jzMQBRIb7BGudzdM7phMbQnygomM0YKNpuqmbyF6+/AJZZW7HnrluuqJs7opfluGLnEE4b6sbTR+ZQrLqYKtkwtflP9GtpF1gwrEHu3T+JC7YM1HRwtuZ8BgJnnxpNLdaUVM/GDySTy0y8+Ixh7NrQs+jrUCO1/ceCziaZ4KaOH9unJ4e/vP7C2LKkBlC2PViJKJJsxoDrC5Sqi/AxaCPUnGXgf7/xIgDAHfeNhcv1fAI14lZ5AeoeGgZhq5zTWE3+U68Nap9njwf3RM0Lrbjp8u246fLt4ffuXCastTPUk8P/ufEFDa/pgi0DuGDLQOwajxdtVJxgMh1VPgMINMOXn7UNb3xhUBjxgg9+KzSVNBrhhlVb7UAwzEk/gtJW9aqjQGP/lApJTRvJRxFcQSfck80gY1CqKSk5c5xuSlLmnmLFRTklOOGcTX1IQ1WPBWp9EMq3l7OM8Hl9/1cfQbHqYmI6mKK0K5aT0V6Cob30F2ZBZkoObvzkPbjzgUM161Qeg2UEP+u67lw4mlp8VFJ6xwzUjuw/8oYL8Y6XnrGIqwjYIk0rvzgyByBhogpNSQu31zIJuuxSdWqS4apAkEjXbLhqPQd80pSkbP2qo0v6GHTqaULReYJ2PnV4DgXLjDko01Dmt2avqfZ8gSlJtX19g4TFoZ5cqN0t5GMAovswLxP1MqYB0yDYXjw8tpF/KieT2NJ8Q3nNTKQGMAXLrAlXDY8Vm6NaDxUNBMO8zOZfSiet3yv9udMTAYPwbAfjM2WM9ucTz1d7dcXt1RpmQcoynT5ZOA2IT+0JBFEaNYJhkaaktA5Hz1BuZaSj4vuflMXS+lIcys2YfYgo1g7lCM5pL5t6WR1PNP0SxnwMmfSXWDclqeiYRoX6Gt1XfflUycG2dV0LTtSiOtWl/g5qLgsVqqrPN5wcBa/rzkaCYYEieoA+9WsUeZQ1jbBWUnJO69RjWcF8CsWqW+t8zkampDATORslZgZTe0b76Nem+wxyMqig2IJg0LWeer4DFbTx3PESRgcKsKSgBDhclWkR1fmnlYLWp/YEEhrDEn0MaZ1oWqjoUlARH8pskpbHsJTRvSJraj6CBnkZzRwzlxKlpWaR053PQPTbpN2bRvcViJd8UOanRihtZam1+5XPQ5nB9PmGkyaedd3ZsHpoQx+D0hjsSGNQAixnGeGz2JQpyYrMRfV8DHp1Un0bVXZbobShoMhksJ2aE1zl1KT5GJpBF6K6+U8/vxKAh2cr2KRN3wm0X1QSC4Y1ht1QMMTrIQ11Z8OEnqVGJTVjSmqFzQOFcISXZkpqPudAJm5pnUzM+axdd/PO53iWbfg5LIMQL5mdNCWl3ZtkYlQSfflpQwsLhrBc9xIFdE5Wc1WOZ31SnmSHPaTNKNZIY0iW857Taj6pfBKgOVOSngmezGNQ1zxVijQGdf9Mg5BJCgbZYectMxRs6jp68kEuSuDLWPy91O+Vbo7LpggGIJpzPDLZtldX3F6tYRakkcbgegIGIVRP13VnMVdxYbv+khPcUgVDnQzlpaDMSfoczgBqXvSFUO0YrqPG69fdbJv1UbjeESZj17trTEkyXDVNMGQaCzzLjCqoNiMYWjUlFbJBVJJquzK3dGfjcyQAiE012YyPQa/aqqY2VWHDQCQYGpmS1Hl8UfssqO9VN3LmK+GhBkD6b6hqVeUts+Z3UM7niuMvafRuGhRWUtVNVvr5dRPsaFhqozWNb6VgwbDGcFyVXVs73aTj+bHOVb3I0yV70UX0oqigxqakVuOvVWy//tIAi1ex1XXpJQiSUUmKpgVDHY0hWW+oK2uCKBIM6l6n3bvQFt6gDWpdM6akVjWGvCwVoqJoNvanF+ID4oIh1+B86hrLdjDtZsn2wmdGhQ0DkY+hocbQwARYiEX1yPBdFU0nv+vtVB12wTJrgitUhdzFlGVPou6ZrjHo59e1ClUmpZnnYTVgwbDGUKak5LSaQOBjyKYIhsl5e9GmpFyKzVahd0YLOUcXYkvdWjRL0xhG6qjxuZQRf7PHrNlfvswqW5mI0J2NSm9XHA9E6fe6GdOBOu9pixEMLfgYgCgXQHWeaaP4Ib3oW4PzqTDMsmaiCp3PstopEAhSoqgzTyNZTTXW9kyt0CgkNAb9N1AddiAY4r4eVZ5lMWXZk6h7tj6mMdQxJQ20tymJ8xjWGI2dzz4yWiKOEgwn5u2gDLXZfEfeaCSjZnFbjlHOpoF0waDO38jJqaNeLD1pqJ4pSZ/4p+ExZceTkfbq6FxxUxKgCqSp+X69cC7gesdspAkVssE8CVsGV96UlCxZrgRDWpmKdd3pI+F6xyw7Xk0Sm6pyCwS+h55spiZPRieuFdTXGJIlKtTvrbdzXXcWpkHIZ80aH5YqzyKEu2Qhq+5ZPCopOlboZ8kYob9msb60k0V7iSlmQVQF1Ypba0py/bgpaSihMTRrRgIa+xjULG6tOp4B3ZTUuHrlQoQaQz3BoHfsTb74Kj8ieQ/CUaZm/gpKb0uNoUGROTUabdQGNVd0M79Xd6umJFWAUGoMG0ONodaUFHc+N+djSDqYdVNSvUl64u2rb0rKpZgH1X+1Tu+YC1lTljQxou0zkSkJCMyAzQ4ckvTmLRQsMzavSDJcFQA29edjJUnSrm21YcGwxgg1BrtWY7BdkepjmFqCYFhoJKPKT7fKpjqCYTEJbkD0gg/XMSXFfAxNmgqICLmMWdPpZk0jqBZaozFEeQz1onaa0Rj6CxZ2jHTXXa+jwi5b1RhOzNvIGISubHC9ydLdwCKcz5qPIQxJ1XwMsTmtFyilEjclxe9pkL8S9xWobbKZuIAAAhOXKoKYT/ggYgEVS/YxqFpadQSDvFZVVDFY32EaAxF9kIgOyfmeHySi12jr3ktE+4joKSJ6lbb8EiJ6RK77a2rVgH0KEoaruilRSb4fq+ky0JUFUaQxpNV7qcdCts+e3PIIhuGeLHIZo2aEOtyTw8a+PHbJQm4LkWZKiuUxpNijmz1uMmKEiPDffuUcXH/JlnBZtzaLWyWlRHSynY06nw//+gX4s9ed31T7lNO+WZNbbXuUKcmWTnTCaH8hFlmjiDmfG4WrZuqbks4Z7cPDh2bw2PhMWCqjcfsa+4YKCYFQqNEY4vtfun0dzt8yUKNhpFX2XSzXX7IF73jpGbH99WdH1W5S/gV1Lsukmgiw1WalfQwfFUL8lb6AiM4BcCOAcwFsAvAdItothPAAfBzAzQDuAfANANcC+OYKt3FNoTSGcorGEPgYohfBNAgDBQsn5qs1U10uRFrCkM5CI71mISL856t24nytfg8QdLT3/Nermz5OGK7akx45kzb7VrPHTbOn/9YVp9e095lj0/jgXY/h8YnZuh3/YFcWWdNo2PnsGGm+7pRKcGuUV9CIsGT5fDU0p/zf331RqvM5b5noyppBWHSDjswwKJwWdT6RxPafXrYTX957EB/46qM4MW8v6Edp5GMAgt9yCrUJbur3JllM0XaDMFRV70rNnaH7GBRLFbIvP3M9Xn4mcGQ2mr8i+ez8zhWn4/IzhqJryhhtl/UMrI7z+ToAXxRCVAE8S0T7AFxKRM8B6BNC3A0ARPRZAK8FC4YYTgONwfFEzfSa67qzODFvBy/IIqbeNAzCn73ufFy2Y13q+je9cBt8+XK1yi1X7Wr5GJFg0DWG9KikxWg6esx7I9Z1ZXFktorP3/s8bM/HS3ePpG73phdtw2U7hhYlpBvR06LzWQnSsRNlbJN5E3p58CTrurOp5ViSFLImKrYXVvdV7ezvsvDHrzoL//XORwAAv3rhpsbHaeBjAHQTkhSQ2bjGoD7bro+ubNTdBWZCIxSouq+jVXu//lsk37lbX31W7PsFWwdC/047sdKC4RYiugnAXgDvEUJMAdiMQCNQjMlljvycXF4DEd2MQLPAtm3bVqDZ7UtaHsMnf7QftufLPIb4SG5IlsUIJppZ3AP/Gy+qf2/f8MKtizrWSqNGZvXCVZeqMeQyRlOO3T98xW788gWjuGzHEI4Vq3VNJD25DM7f0p+6binsGOnBzvU9OHu0OZNbknNG+7DntEHsPTAVJmg1Yqg7m5pDk0TNyZDUGADgTZduxVmjvTh9qDsslV2PmPM5pX1Jk1DSlKQ+zwGxaqYAMNBlRdOhLlOZF2jnsUxqqFkBwFsuOw1vuey0ls63ErQkGIjoOwA2pqx6HwKz0J8CEPL/RwD8DoC0OyUaLK9dKMRtAG4DgD179izPsHWNYKc4n//lgUMwKDBTpGkM+48XY7XpT0X2nLYO+84qQk0Ik3S2xzSGRUSddGXNpmzOG/vzYTmJRiPu5WZddzY2n8NiISL8/jW78JZP/Sw2om50vmZGuAUrmIchLYmNiHDxtsF6u8bIZYLQXZGS+QzUhlWrTjn+2wcJiEm/yD+97UXRXN7LqDFYpgHLpLbLZl4MLQkGIcQ1zWxHRJ8E8K/y6xgAfbi5BcC4XL4lZTmj4aaYksamSljXnUV3LhOWU1AMdmdx4oCNoe5crF7QqcYrztmAV5yzAUBUkTMtj6Fe4lk93vuas2vu6anGL+0cxsvPHMEZTfg2Xn3+KM6U82c0IiznbbvIJubGWAxEFAqZ9Gq1cadzFK6qO32N1LyS3Vpggy64liOoIm+Zi3rO2o0VMyUR0agQYkJ+fR2AR+XnuwB8noj+NwLn8y4APxNCeEQ0R0SXAbgXwE0A/mal2rdWURVUHU/A9YKqmHMVF1nTwFC3XxMXPiR9DBXXC+vVnOqomvwxU5IZdSCLCXZ74fZ0H8upBBHhH3/rhU3dlzfsac6EqCbYKVbc1NDXxaAEQ5rmple61b8n5+JImpGSpFX2bYWurAlzDQdVrmRP8ZdEdBECc9BzAN4OAEKIx4joywAeB+ACeJeMSAKAdwL4DIACAqczO54T6DNRVVwfB6dKAIIsUscTNaPbzYNB9dLnjs/jBU2q72udMGtaC1dVk8S0WyJRu7DckeFBZ+42lcS2EI2SwJLrCmmmJGthc6ApczhKKXNLL4XFDkDajRUTDEKItzRY9yEAH0pZvhfAeSvVplMBRxcMjoexqWDiFNv1MW+7GDXj8ec7hoNEqamSs6ZV28UQmhNSEtPaLZHoVCVvmZict2OT9CyVRuVZklVV6zmfmxkQqGlSl2PwUMhm4Ptr1/3ZGT3FKYQuGMq2hzGpMQDBhCVJ57MeE2+dws5nHZUtnRSE2SYjjJjWKWSjKUNbFgwyoCAtCSw0JSUyiHWNYbDLwpBW56kevS2G/sbbZTSsJ9XudIbR+RRC+RgAoOpGGgMQlNdOhqsO92TRm8+EfohOoJA1kTFqQwWzmcaJZczyUbCMsCSGmgdh6ccy6ybwhfN2JP7rzuc/ve48eE3k3KikzeXQGLat6wqryK5FWDCsMWxXNyX5McHgC8Qyn4HAdrxjpAcPHZw+pcNVddQIM0mzJgWmdaI8Bg/dw611MzmrvkBXAqNedVUgXga7ESo4Yzk0hr+8/sKWj7GadEZPcQrh+popyQlMSXpiUtKUBER+hlYn1Vkr5C0z9VoDUxILhpNBPmuibHtN1UNaiIJl1hXoYbVabV4FyyQMdtVWh10IvdBfq7QSotsOsMawxlCZz0DkfD5zYy/uf34aAFIL5SnBsJYf1MUw0GWhr1DbMazvzaUWh2OWnx3D3bA9H8eL1bAC7FJ5w56tmJgpp67rSmgKvXkL//qfX4LtwwvPZZGkN5dBLmMsmK3cCbBgWGPozufDMxUUqy7OHu3TBEOKxiAd0J3iY/j9q3fjpsu31yz/+9/cA3MRFWaZpXP9JVvxhZ8dxIMHp2umbV0s18jExTRec8EoiChWCuXMjUsrD7KhPx+rtdXJdEZPcQqh5zE8IzNQz9JehHTB0Fkaw0hvLpbVqujvslo2azDNYRqEv7rhQvTkMtg2tHIlQtb35vHWF29flpyBW16+E//8jsuXoVVrH35L1hiO54eTwhyYDATDGeu1kNSUEfHpw90Y7LLCSXEY5mSwc30P9r7/mjXj1+nOZVpOxjtV4LuwxnA8IScud3FgMshh2LauCxmD4Pq1ZbeBwBn701uv7hjnM9M+rBWhwMThnmKN4Xg++uRsZ8+fCATD+t58GFGRqWNDL2RNdqoxDNMULBjWGI7nh7Xji1UX67qzyGaMcFmnOJgZhlk5uBdZYzieQM4yQl/C+l5VTz7QIk71EtEMw6w8LBjWGMEsbdE8sSouX9V56ZR6SAzDrBzci6wxbFcKBpnxGWoM0pRkGfyTMgzTGtyLrDEcz5flo4OfTmkMPaHGwKYkhmFagwXDGsPxBCyTNFNSoDEo53OGNQaGYVqEe5E1hvIxqGqTI7KkcWhK4qgkhmFahHuRNYbjCVgZo1ZjUKYkrgXEMEyLrJhgIKIvEdGD8u85InpQLt9ORGVt3Se0fS4hokeIaB8R/TWt5UlTVwjH82EZFM4OVeNjYI2BYZgWWck5n9+oPhPRRwDMaKufEUJclLLbxwHcDOAeAN8AcC2Ab65UG9cioSlJlhpQ1SB7ZDZ0vcxnhmGYZlnx4aUc9b8BwBcW2G4UQJ8Q4m4hhADwWQCvXen2NeLobAVH5yqr2YQaHM+HJaeoVFnPQKQxcOYzwzCtcjKK6L0EwBEhxNPastOJ6AEAswDeL4T4MYDNAMa0bcbkshqI6GYEmgW2bdu2Io0GgD++42FkMwY+edOeFTvHYhBCyKgkA79x6TZcccZwuO7ibQN46e4R7Fq/tFr0DMMwipYEAxF9B8DGlFXvE0J8TX5+E+LawgSAbUKISSK6BMBXiehcAGk2kNQZvIUQtwG4DQD27Nmz8CzfS2Sm7LRVRVLHCy41axJetGMIL9oxFK5b35fH7b9z6Wo1jWGYU4iWBIMQ4ppG64koA+D1AC7R9qkCqMrP9xHRMwB2I9AQtmi7bwEw3kr7WsXzBTx/xeTOolGzt7GDmWGYlWSle5hrADwphAhNREQ0QkSm/LwDwC4A+4UQEwDmiOgy6Ze4CcDX0g56snB9AU+0j2BwpcbAgoFhmJVkpX0MN6LW6XwlgP+PiFwAHoB3CCFOyHXvBPAZAAUE0UirGpHk+wJ+G2kMalpPLpTHMMxKsqKCQQjxWynLvgLgK3W23wvgvJVs02JwfR+eaJ/wz9CUxKW1GYZZQXjo2YDAx7DarYhgHwPDMCcD7mEa4LaZKclhUxLDMCcB7mEa4PkCrt8+KoPtRuGqDMMwKwULhga4vkAbKQxsSmIY5qRwMjKf1yztlMdw421348wNQVYzCwaGYVYSFgwNcD2/LQRDyXZxz/4TmJgJ6jaxYGAYZiVhwdAAzxfwVzHB7duPHcbmwQL6ZOXUQ1NlAECWp+9kGGYF4aFnA1xfwF1FjeFPvvIwPv6DZ3C8WA3bA/D0nQzDrCzcwzTAW8Vw1WLVxVTJwdHZKiaLdmwdm5IYhllJuIdpgCdWr1aSMhsdmatgcr4aW8emJIZhVhIWDHXwfQEhUNf5fGByHv9y/1jquuXg0HQJAHBktoLjrDEwDHMS4R6mDsqeX8+U9OW9B/HHdzwcW3ZktgKxTBrGmNQYKo6PZ4/Px9axYGAYZiXhHqYOSlOo53wu234sz+HA5Dwu//Pv4oe/OLYs51eCAQAeH5+NTRjEgoFhmJWEe5g6qFIYKlz1G49M4IHnp8L1FdcDEGUjP3JoBr4Anjw8tyznP6QJhqePzuHMjb0g6VrgeZ0ZhllJuIepg9IE1P8//+YTuP2nz4XrK04gGNQcCU8fKQIADp4oLcv5x6ZK2DJYABBM6bmhL4/1vTkAQIZrJTEMs4KwYKhD6GMQgBACrifCOZcBoOoGAsGR//cdlYJBG+m3wqHpMi7eNhh+H+7JYmN/ICjYlMQwzErCPUwd9GgkXwSCwtEmZ6g6ypQUbPf00cCENDbVusZQtj0cL9o4c2MvenJBcvpQdw6b+vMAAIs1BoZhVhAWDHXQnc4q0U1fVnGkxuD5cLwgcogocBq3mhSnQlW3DBawvi8wHw31ZLFlsICCZYKIBQPDMCtHS4KBiG4goseIyCeiPYl17yWifUT0FBG9Slt+CRE9Itf9NclejohyRPQlufxeItreSttaxfPigqFGY9CczwcmS3A8gYu2DsB2fRwrVmuOtxhURNLmgQI29AZawnBPDr935Q586q17Gu3KMAzTMq1qDI8CeD2AH+kLiegcADcCOBfAtQD+johMufrjAG4GsEv+XSuXvw3AlBBiJ4CPAvhwi21rCT3j2RNBWKrrpWkMAk8fCcxIV525HkDrDuhjc4FgWd+bj2kM63vzePHO4ZaOzTAMsxAtCQYhxBNCiKdSVl0H4ItCiKoQ4lkA+wBcSkSjAPqEEHeLIBPsswBeq+1zu/x8B4CraRVtJp42c5uayU2fza3iRBrD09Lx/DIpGMZadEBPlxwAwEC3hQ19kcbAMAxzMlgpH8NmAAe172Ny2Wb5Obk8to8QwgUwA2Ao7eBEdDMR7SWivceOLU9CWRLdn+DLRLa0qCTb83HwRAkb+nLYtaEHQOsaw1TJRsYg9OYy2DpYgGlQGKrKMAyz0iw4HwMRfQfAxpRV7xNCfK3ebinLRIPljfapXSjEbQBuA4A9e/asSJU73WzkCSFLcNdqDK4nUHV9FCwTecvESG8OB1uMTJoqORjoskBEuGHPVly0dRADXdmWjskwDNMsCwoGIcQ1SzjuGICt2vctAMbl8i0py/V9xogoA6AfwIklnLtphBC49v/8GO+6aid+7cJNsXV6uKrrBQX13LQ8Bs+H7frIypIVWwcLOHiiVVOSHQqCvGXi/C39LR2PYRhmMayUKekuADfKSKPTETiZfyaEmAAwR0SXSf/BTQC+pu3zVvn5egDfE8tVka4OtufjqSNz2HektoyFbkqyNSGg0DOfbS8SDOu6s5itOC21a6pkY7DLaukYDMMwS6XVcNXXEdEYgMsBfJ2IvgUAQojHAHwZwOMA/g3Au4QQntztnQD+AYFD+hkA35TLPwVgiIj2AXg3gFtbaVszKK2gqnX4yXUAYHvSbCSXCSFimc+264f1i/KWibLjoRWmSw6bjhiGWTVamvNZCHEngDvrrPsQgA+lLN8L4LyU5RUAN7TSnsWinMmOW6uY6P4EJQSUKUl9D7YTMVNSwTJRsVsTDFMlGxew+YhhmFWiozOfXa/WRKTwGpiSqk60vROakoI0jUK2NY1BCIGpkoNB1hgYhlklOlswyM4/TTCk+RjUMlVyW63TTUmFFk1JZceD7fpsSmIYZtXoaMGgBIKdpjF4uo+hkcYgYHt+OJFO3jJRcfxF1Us6eKKEJyZmAQShqgDY+cwwzKrR0YLBCzWG2k5cL4lhJ3wMusagwlVVxdNCNjAp6X6IhfiLf3sSf/DFBwEAU/PB/M6D3awxMAyzOnS0YIicz835GJRDuuLUCgbd+QxgUeakmZKDI3MVAIHjGQD7GBiGWTU6WjCojn5BH0NoShKxUFW1TM9jyFvB/8UIhnnbxXTJgeP5bEpiGGbV6WzBIDWGVB+DHq6q+RQ8X6RrDGagKeSVxrCIkNVSNdh2at7GtNQY2PnMMMxq0dmCoVFUUorzWe1TE66aYkqqLEJjKDkuAOB40cbUvKysyhoDwzCrRGcLBs1EVHE83Hjb3Xj00AyAdB9DsK0fD1dNlMRQzudFCQapMUzOVzFVstGby/C8zgzDrBod3fuEzmfPx9HZKu7ZfwIPHpwGkO5jAAJNoqJpDMpklGvB+TxvBxrDZDEwJQ10s7bAMMzq0dGCQTmfbdcP6yEps1JdjcH3w2k9AaBYDTp1vVYS0LyPIfBZBMc/Xqxy1jPDMKtOZwsGzfmsOudklrO+TO2jts1lDMxLwZDMY2hWYyhJbQEAJudtHJ2rYohzGBiGWUU6WzBozudqoh6SHpVUa0oKOv3evIV56R8IayUt0vlc0jSLo7NV7D9WxBkjPUu6HoZhmOWgswWDcj67ItQKFtIYAlOSDyKgO2eG/oGaBLcmTUm6YHh4bBpV1w+nCGUYhlkNOlowODGNQU28EyzTax1VE6akquMhlzGQNSNTUjIqqezUhsCmofYHgKePFgEAO9ezYGAYZvXoaMHgakX0kqW162oM0uyUy5iwTCMyJUnns4pOat7HEGw30psLl+0c6V3S9TAMwywHnS0YUnwMSgh49cJVZeZz3jJgmRRGJSmBQETBZD1NCgZlitq2rgsAsL43h35ObmMYZhXpbMHgRdVVk85nXWOoap286/lSMAQaQynhYwDkZD3N+hikxqEEA5uRGIZZbVqd8/kGInqMiHwi2qMtfwUR3UdEj8j/V2nrfkBETxHRg/JvvVyeI6IvEdE+IrqXiLa30rZmUHkMni9C04+dksegl8xQQiSXMWCZRpgkFxMMi5isR2kMW6Vg2MWCgWGYVaalOZ8BPArg9QD+PrH8OIBfFUKME9F5AL4FYLO2/s1y7medtwGYEkLsJKIbAXwYwBtbbF9D9HkYlBM4OfcCkDQlaRqDJgz0EhZ5y2haMCjNItQYNrB/gWGY1aUljUEI8YQQ4qmU5Q8IIcbl18cA5Ikol9wuwXUAbpef7wBwNRFRK+1bCFfr8JVgSM1jSElwy2dMZM2oeVkzbkqqNGlKUhrDC7YNYKg7ixedvm4JV8IwDLN8nAwfw68DeEAIUdWWfVqakT6gdf6bARwEACGEC2AGwFDaAYnoZiLaS0R7jx07tuSG6X6EuUpCY2gYleQhZxnIGNHt001J+UzzpqRS1YNpEHYMd+O+D7wCu1ljYBhmlVlQMBDRd4jo0ZS/65rY91wEJqG3a4vfLIQ4H8BL5N9b1OYph0idOFkIcZsQYo8QYs/IyMhCzaiLbi4qhhpDsMzzRVjmIpbHIGsb5TJxU1Iu4XxeTFRSl2VihZUjhmGYplnQxyCEuGYpByaiLQDuBHCTEOIZ7XiH5P85Ivo8gEsBfBbAGICtAMaIKAOgH8CJpZy7WVy/1pRka1FJWdOA43mpZbdzMlxVEdMYLBPHi3ZTbShVPXTlzJaug2EYZjlZEVMSEQ0A+DqA9wohfqItzxDRsPxsAfgVBA5sALgLwFvl5+sBfE8IkaoxLBe68zlpSvJ9EXb2yVpJ1dDHoJmSzHhU0mI0hu5sqzEADMMwy0er4aqvI6IxAJcD+DoRfUuuugXATgAfSISl5gB8i4geBvAggEMAPin3+RSAISLaB+DdAG5tpW3NoDuY5xLOZ9cXyMnCeDHnsyy7HSS4pfsYCtYi8hhs1hgYhmkvWhqqCiHuRGAuSi7/nwD+Z53dLqlzrAqAG1ppz2JpFK7q1dEYHKkx5DImhOYCqUlwW0TZ7S7WGBiGaSM6O/NZ0xiKlaTGEE3XGQ9X9VGWJTF081HGiPwN+UUkuJVsD91Z1hgYhmkfOlswpGgMelSS6vj1qKSy48P1BbqyZmhKymaMWFRRwTJhu34se7oe81UXXTnWGBiGaR86WjDopqSiTDSrankMlkkgipfHmK04AIBCNhMKhpwZv42FbPC9GQc0awwMw7QbHS0YdOezin/S53w2DYqZiABgthwIhq6sCSsTrNP9C4A2WU8TgmG+yj4GhmHai44WDE6KqUevlZQxDBjSRKRyFmalL6Ira8IyIlOSTn4Rs7iVbA9drDEwDNNGdPRQ1fV8mAalVlL1fAHDAEypMQTlL/xQYyhYZigsajSGbHPzPttu4K/oZh8DwzBtREdrDK4n0GXFR+uuL+D7Aq7vI2MYMEkJBkLGMEIfQ1c2E5bEyCZ9DE2aktRcDqwxMAzTTnS2YPBFanKZ7fmhj8GQGoNpEjImRRpDIipJR/kMitp8zmmo9Zz5zDBMO9HhgsFPdfw6ng9PCGQMCk1JJhEs04j5GJSmYCU0hgE5Ned0yWl4/ql5J7Y9wzBMO9DRQ1XHE6GjGAAMAnwRLHe9QGMIBYP8P6NFJWXq+BgGu7IAgKlS40J6k/NBJfKhnuwyXA3DMMzy0Nkag+fH7PvKCayS0zImxXwMlmmEeQ66KSmXWZrGcGI+EBzruheaw4hhGObk0dmCQWYwK3qlYHBCH4MRaQzSx6DoymZCU1LS+Zy3TBQsE1PzjTUGJRhYY2AYpp3obMHgBRVUVTWL3nww0q/KMNKMQVCTtGUMI5bsFoSrpjufAWCwy8KUpjHcs38SY1Ol2DbHizYsk0KBxDAM0w50tmDwfVgmhR18t4xQcrSoJGVKMg2KCYLge7qPAQAGurKY1nwM7/rc/fjY95+JbXNivop13VmevY1hmLaiswWDdDArU1CP1Bgcz5d5DFq4KkWmJGV+ytQxJQHAYLcVcz7PVhwcninHtjkxb7N/gWGYtqOjBYPj+7DMaIrO3nzc+azXSjJlghuAMCku28CUFGgMTng8xxM4MluNbTM5b2OY/QsMw7QZHS0YPE+E0UZA5Hy2Pc3HoKKSzMh0pEpeqCJ6yTwGQPkYAo1B1Uw6OleJbTNZtLGumwUDwzDtRUcLBscXyJjRFJ09eriql4hK0jUGmRRXL1wVCHIZZsoOfF+g5ARJcZPzdliLCVCmJBYMDMO0F63O+XwDET1GRD4R7dGWbyeisjbf8ye0dZcQ0SNEtI+I/pqk55WIckT0Jbn8XiLa3krbmsH1AuezMgV1h+GqItAYTNKK6EU+BqUxNDIlDXZl4YvAt1CSGoMQwPFiYE6qOB6KVRdDLBgYhmkzWtUYHgXwegA/Sln3jBDiIvn3Dm35xwHcDGCX/LtWLn8bgCkhxE4AHwXw4RbbtiDK+Zz0MaiSGAZFpiQ9KilyPsuopDrOZwCYKjmx8tvKzxDlMLDzmWGY9qIlwSCEeEII8VSz2xPRKIA+IcTdQggB4LMAXitXXwfgdvn5DgBX0wrHcSrnsxrxJ53Peq0kPY9BCYZGeQwDWlmMea2Y3tHZwM8QZT2zxsAwTHuxkj6G04noASL6IRG9RC7bDGBM22ZMLlPrDgKAEMIFMANgKO3ARHQzEe0lor3Hjh1bcgNV5x/lMdRGJSnBYGjbFaxgu55cBv0FC1vXddUcW9VLmi7ZKGnlt4/MBRrDpNIYWDAwDNNmLJhyS0TfAbAxZdX7hBBfq7PbBIBtQohJIroEwFeJ6FwAaRqAmiWn0br4QiFuA3AbAOzZsyd1m4UQQsDx0p3Pah6FjBGvlZTMY8hbJva+/5qa6T+BICoJCCqoFrSyG5HGEAgI1hgYhmk3FhQMQohrFntQIUQVQFV+vo+IngGwG4GGsEXbdAuAcfl5DMBWAGNElAHQD+DEYs/dLGrWNktLcFMlMZRgME1KjUrSO/q0UFUgbkryRVRW+4gUDJNF9jEwDNOerIgpiYhGiMiUn3cgcDLvF0JMAJgjosuk/+AmAErruAvAW+Xn6wF8T/ohVgRXCgbTrHU+K2exnvmc0ZzUBWvhGdf68hmYBmG65ISCZlN/Hkc1U5JlEvryXCeJYZj2otVw1dcR0RiAywF8nYi+JVddCeBhInoIgSP5HUIINfp/J4B/ALAPwDMAvimXfwrAEBHtA/BuALe20raFcEONob4pyTQMqIKqZoopqRFEhIFCkOSmwlW3D3eHUUnTJRv9Ba6TxDBM+9HScFUIcSeAO1OWfwXAV+rssxfAeSnLKwBuaKU9i8GViWYZk8K5m9UUm7rGYErzUSaW4NbcHM0DXRamSw6GuoOopNOGuvDU4SMAgNmKi74CawsMw7QfHZv57HiBxpAxjdDHkLOCz2qEH0QlBdsbuimpyTmaB7uyocZQsExs6MuH2c+zZQd9eZ7Sk2GY9qNjBYPrBxqDpZfPljkNFT0qKZb5vFiNIYupkoOS46E7Z4YRSDNlR2oMLBgYhmk/OlcwSI1BZTRbJoVaQcWJNIYo89mAZcRLYizEYJeF6ZKNsu2hkDXRX4im/JyrOOx4ZhimLelcwaCcz6aBnnwmDFW1NFNSba2keNnthRjszoaZz11WJhQMM2Ubs2U3PCfDMEw70bFDVt35/Hsv2YFfPn8UQFDeIh6VpOUxhFFJzd22gS4LFcfHVMlGV84McxsCU5LDzmeGYdqSju2ZQuezYWC4J4dhmWiWNev7GKyUBLdGqLIYh6bKOH2kO9QYjsxWYbs+O58ZhmlLOtiUJJ3PZjyPQDclGZTIfF5EHgMQlcU4PFtBwcpgQAqGgydKAMA+BoZh2pIOFgyR81lHNyXF5nw2CBduHcDlO4ZC7WIhlOnIF4EwUVFIB6eCuZ85KolhmHakY4esKiopWevIMiksiT3QZcWK6F28bRBfuPmyps+hTEkA0J0zYRqE3nxG0xhYMDAM0350rsagnM8pGoPnCwx2Wbho64BmSlr8rVKmJCAq1T3QZUWCgZ3PDMO0IR0rGBw/ynzWURrE1WdvQMY0wjyGjLn4mkYDmsag/BIDhWw4FwOHqzIM0450rGCoqzFIwfDKczYE683Ix7BYshkD3VIgdOWC//2aX4FNSQzDtCOdKxhCjaHWlJS3DLxk1wgARJnPS6yCqrQGlRTXr5mX2JTEMEw70rE9Uz3n802Xb8crztkQ5iqo1UvRGABgsNvCoelymBSnQlYzBjU1rwPDMMzJpnMFg59uSrr8jPg002YLPgYgikxSgkaZknrzGZ6LgWGYtqRjTUlOHY0hiZ7HsBSUKalb+hgGpCmJcxgYhmlXOlYweFJjWKjD1/MYloIKWQ3DVQuBoGDHM8Mw7UrHCoZoop4FBIO59DwGQHM+S1OS0hTY8cwwTLvS6pzPNxDRY0TkE9EebfmbiehB7c8noovkuh8Q0VPauvVyeY6IvkRE+4joXiLa3krbFkKFq1oLdPjLpTEkTUm9OdYYGIZpT1rVGB4F8HoAP9IXCiE+J4S4SAhxEYC3AHhOCPGgtsmb1XohxFG57G0ApoQQOwF8FMCHW2xbQ+qFqyZRpiZjiYLh4m2DOHNDL0b7CwAi5zNrDAzDtCstCQYhxBNCiKcW2OxNAL7QxOGuA3C7/HwHgKtpBcN29Il6GmG0qDFcuHUA3/rDK9Gdi0piAOxjYBimfTkZPoY3olYwfFqakT6gdf6bARwEACGEC2AGwBBSIKKbiWgvEe09duzYkhr19it34Jk/ew1ymQVMSS1GJSUZKGRhUDC7G8MwTDuyoD2DiL4DYGPKqvcJIb62wL4vAlASQjyqLX6zEOIQEfUC+AoCU9NnAaT1vCLtuEKI2wDcBgB79uxJ3WYhiAjNpCboE/UsB4Wsic/89qU4f3P/shyPYRhmuVlQMAghrmnh+DcioS0IIQ7J/3NE9HkAlyIQDGMAtgIYI6IMgH4AJ1o497Kw3BoDAFy5e2TZjsUwDLPcrJgpiYgMADcA+KK2LENEw/KzBeBXEDiwAeAuAG+Vn68H8D0hxJK0geUkikrq2MhehmE6jJZCY4jodQD+BsAIgK8T0YNCiFfJ1VcCGBNC7Nd2yQH4lhQKJoDvAPikXPcpAP9ERPsQaAo3ttK25cIIo5JWuSEMwzAniZYEgxDiTgB31ln3AwCXJZbNA7ikzvYVBBpGW6GCllhjYBimU+DebgHCstvL6GNgGIZpZ1gwLIDSFJYrKolhGKbdYcGwACsRlcQwDNPOsGBYgMvPGMI7XnoGztzYu9pNYRiGOSlwwZ4F6C9YuPXVZ612MxiGYU4arDEwDMMwMVgwMAzDMDFYMDAMwzAxWDAwDMMwMVgwMAzDMDFYMDAMwzAxWDAwDMMwMVgwMAzDMDGoDaY8aAkiOgbgwBJ3HwZwfBmbcyrC96gxfH8Whu9RY1br/pwmhEidNWzNC4ZWIKK9Qog9q92OdobvUWP4/iwM36PGtOP9YVMSwzAME4MFA8MwDBOj0wXDbavdgDUA36PG8P1ZGL5HjWm7+9PRPgaGYRimlk7XGBiGYZgELBgYhmGYGB0rGIjoWiJ6ioj2EdGtq92edoCIniOiR4joQSLaK5etI6J/J6Kn5f/B1W7nyYSI/pGIjhLRo9qyuveEiN4rn6mniOhVq9Pqk0ed+/NBIjokn6MHieg12rpOuz9biej7RPQEET1GRL8vl7f1M9SRgoGITAAfA/BqAOcAeBMRnbO6rWobXi6EuEiLq74VwHeFELsAfFd+7yQ+A+DaxLLUeyKfoRsBnCv3+Tv5rJ3KfAa19wcAPiqfo4uEEN8AOvb+uADeI4Q4G8BlAN4l70NbP0MdKRgAXApgnxBivxDCBvBFANetcpvalesA3C4/3w7gtavXlJOPEOJHAE4kFte7J9cB+KIQoiqEeBbAPgTP2ilLnftTj068PxNCiPvl5zkATwDYjDZ/hjpVMGwGcFD7PiaXdToCwLeJ6D4iulku2yCEmACChxzA+lVrXftQ757wcxVxCxE9LE1NykzS0feHiLYDeAGAe9Hmz1CnCgZKWcZxu8AVQoiLEZjY3kVEV652g9YY/FwFfBzAGQAuAjAB4CNyecfeHyLqAfAVAH8ghJhttGnKspN+jzpVMIwB2Kp93wJgfJXa0jYIIcbl/6MA7kSgwh4holEAkP+Prl4L24Z694SfKwBCiCNCCE8I4QP4JCJTSEfeHyKyEAiFzwkh/kUubutnqFMFw88B7CKi04koi8DZc9cqt2lVIaJuIupVnwG8EsCjCO7LW+VmbwXwtdVpYVtR757cBeBGIsoR0ekAdgH42Sq0b1VRHZ7kdQieI6AD7w8REYBPAXhCCPG/tVVt/QxlTvYJ2wEhhEtEtwD4FgATwD8KIR5b5WatNhsA3Bk8x8gA+LwQ4t+I6OcAvkxEbwPwPIAbVrGNJx0i+gKAlwEYJqIxAP8dwF8g5Z4IIR4joi8DeBxBNMq7hBDeqjT8JFHn/ryMiC5CYAJ5DsDbgc68PwCuAPAWAI8Q0YNy2X9Fmz9DXBKDYRiGidGppiSGYRimDiwYGIZhmBgsGBiGYZgYLBgYhmGYGCwYGIZhmBgsGBiGYZgYLBgYhmGYGP8/A14SsRzx7TEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -150:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
